toolboxv2 API Reference

This section provides an API reference for key components directly available from the toolboxv2 package.

Core Application & Tooling

toolboxv2.AppType

Source code in toolboxv2/utils/system/types.py

class AppType:
    prefix: str
    id: str
    globals: dict[str, Any] = {"root": dict, }
    locals: dict[str, Any] = {"user": {'app': "self"}, }

    local_test: bool = False
    start_dir: str
    data_dir: str
    config_dir: str
    info_dir: str
    is_server:bool = False

    logger: logging.Logger
    logging_filename: str

    api_allowed_mods_list: list[str] = []

    version: str
    loop: asyncio.AbstractEventLoop

    keys: dict[str, str] = {
        "MACRO": "macro~~~~:",
        "MACRO_C": "m_color~~:",
        "HELPER": "helper~~~:",
        "debug": "debug~~~~:",
        "id": "name-spa~:",
        "st-load": "mute~load:",
        "comm-his": "comm-his~:",
        "develop-mode": "dev~mode~:",
        "provider::": "provider::",
    }

    defaults: dict[str, (bool or dict or dict[str, dict[str, str]] or str or list[str] or list[list]) | None] = {
        "MACRO": list[str],
        "MACRO_C": dict,
        "HELPER": dict,
        "debug": str,
        "id": str,
        "st-load": False,
        "comm-his": list[list],
        "develop-mode": bool,
    }

    cluster_manager: ClusterManager
    root_blob_storage: BlobStorage
    config_fh: FileHandler
    _debug: bool
    flows: dict[str, Callable]
    dev_modi: bool
    functions: dict[str, Any]
    modules: dict[str, Any]

    interface_type: ToolBoxInterfaces
    REFIX: str

    alive: bool
    called_exit: tuple[bool, float]
    args_sto: AppArgs
    system_flag = None
    session = None
    appdata = None
    exit_tasks = []

    enable_profiling: bool = False
    sto = None

    websocket_handlers: dict[str, dict[str, Callable]] = {}
    _rust_ws_bridge: Any = None

    docs_reader: Callable | None = None
    docs_writer: Callable | None = None
    get_update_suggestions: Callable | None = None
    auto_update_docs: Callable | None = None
    source_code_lookup: Callable | None = None

    initial_docs_parse: Callable | None = None

    def __init__(self, prefix=None, args=None):
        self.args_sto = args
        self.prefix = prefix
        self._footprint_start_time = time.time()
        self._process = psutil.Process(os.getpid())

        # Tracking-Daten für Min/Max/Avg
        self._footprint_metrics = {
            'memory': {'max': 0, 'min': float('inf'), 'samples': []},
            'cpu': {'max': 0, 'min': float('inf'), 'samples': []},
            'disk_read': {'max': 0, 'min': float('inf'), 'samples': []},
            'disk_write': {'max': 0, 'min': float('inf'), 'samples': []},
            'network_sent': {'max': 0, 'min': float('inf'), 'samples': []},
            'network_recv': {'max': 0, 'min': float('inf'), 'samples': []},
        }

        # Initial Disk/Network Counters
        try:
            io_counters = self._process.io_counters()
            self._initial_disk_read = io_counters.read_bytes
            self._initial_disk_write = io_counters.write_bytes
        except (AttributeError, OSError):
            self._initial_disk_read = 0
            self._initial_disk_write = 0

        try:
            net_io = psutil.net_io_counters()
            self._initial_network_sent = net_io.bytes_sent
            self._initial_network_recv = net_io.bytes_recv
        except (AttributeError, OSError):
            self._initial_network_sent = 0
            self._initial_network_recv = 0

    def _update_metric_tracking(self, metric_name: str, value: float):
        """Aktualisiert Min/Max/Avg für eine Metrik"""
        metrics = self._footprint_metrics[metric_name]
        metrics['max'] = max(metrics['max'], value)
        metrics['min'] = min(metrics['min'], value)
        metrics['samples'].append(value)

        # Begrenze die Anzahl der Samples (letzte 1000)
        if len(metrics['samples']) > 1000:
            metrics['samples'] = metrics['samples'][-1000:]

    def _get_metric_avg(self, metric_name: str) -> float:
        """Berechnet Durchschnitt einer Metrik"""
        samples = self._footprint_metrics[metric_name]['samples']
        return sum(samples) / len(samples) if samples else 0

    def footprint(self, update_tracking: bool = True) -> FootprintMetrics:
        """
        Erfasst den aktuellen Ressourcen-Footprint der Toolbox-Instanz.

        Args:
            update_tracking: Wenn True, aktualisiert Min/Max/Avg-Tracking

        Returns:
            FootprintMetrics mit allen erfassten Metriken
        """
        current_time = time.time()
        uptime_seconds = current_time - self._footprint_start_time

        # Formatierte Uptime
        uptime_delta = timedelta(seconds=int(uptime_seconds))
        uptime_formatted = str(uptime_delta)

        # Memory Metrics (in MB)
        try:
            mem_info = self._process.memory_info()
            memory_current = mem_info.rss / (1024 * 1024)  # Bytes zu MB
            memory_percent = self._process.memory_percent()

            if update_tracking:
                self._update_metric_tracking('memory', memory_current)

            memory_max = self._footprint_metrics['memory']['max']
            memory_min = self._footprint_metrics['memory']['min']
            if memory_min == float('inf'):
                memory_min = memory_current
        except (psutil.NoSuchProcess, psutil.AccessDenied):
            memory_current = memory_max = memory_min = memory_percent = 0

        # CPU Metrics
        try:
            cpu_percent_current = self._process.cpu_percent(interval=0.1)
            cpu_times = self._process.cpu_times()
            cpu_time_seconds = cpu_times.user + cpu_times.system

            if update_tracking:
                self._update_metric_tracking('cpu', cpu_percent_current)

            cpu_percent_max = self._footprint_metrics['cpu']['max']
            cpu_percent_min = self._footprint_metrics['cpu']['min']
            cpu_percent_avg = self._get_metric_avg('cpu')

            if cpu_percent_min == float('inf'):
                cpu_percent_min = cpu_percent_current
        except (psutil.NoSuchProcess, psutil.AccessDenied):
            cpu_percent_current = cpu_percent_max = 0
            cpu_percent_min = cpu_percent_avg = cpu_time_seconds = 0

        # Disk I/O Metrics (in MB)
        try:
            io_counters = self._process.io_counters()
            disk_read_bytes = io_counters.read_bytes - self._initial_disk_read
            disk_write_bytes = io_counters.write_bytes - self._initial_disk_write

            disk_read_mb = disk_read_bytes / (1024 * 1024)
            disk_write_mb = disk_write_bytes / (1024 * 1024)

            if update_tracking:
                self._update_metric_tracking('disk_read', disk_read_mb)
                self._update_metric_tracking('disk_write', disk_write_mb)

            disk_read_max = self._footprint_metrics['disk_read']['max']
            disk_read_min = self._footprint_metrics['disk_read']['min']
            disk_write_max = self._footprint_metrics['disk_write']['max']
            disk_write_min = self._footprint_metrics['disk_write']['min']

            if disk_read_min == float('inf'):
                disk_read_min = disk_read_mb
            if disk_write_min == float('inf'):
                disk_write_min = disk_write_mb
        except (AttributeError, OSError, psutil.NoSuchProcess, psutil.AccessDenied):
            disk_read_mb = disk_write_mb = 0
            disk_read_max = disk_read_min = disk_write_max = disk_write_min = 0

        # Network I/O Metrics (in MB)
        try:
            net_io = psutil.net_io_counters()
            network_sent_bytes = net_io.bytes_sent - self._initial_network_sent
            network_recv_bytes = net_io.bytes_recv - self._initial_network_recv

            network_sent_mb = network_sent_bytes / (1024 * 1024)
            network_recv_mb = network_recv_bytes / (1024 * 1024)

            if update_tracking:
                self._update_metric_tracking('network_sent', network_sent_mb)
                self._update_metric_tracking('network_recv', network_recv_mb)

            network_sent_max = self._footprint_metrics['network_sent']['max']
            network_sent_min = self._footprint_metrics['network_sent']['min']
            network_recv_max = self._footprint_metrics['network_recv']['max']
            network_recv_min = self._footprint_metrics['network_recv']['min']

            if network_sent_min == float('inf'):
                network_sent_min = network_sent_mb
            if network_recv_min == float('inf'):
                network_recv_min = network_recv_mb
        except (AttributeError, OSError):
            network_sent_mb = network_recv_mb = 0
            network_sent_max = network_sent_min = 0
            network_recv_max = network_recv_min = 0

        # Process Info
        try:
            process_id = self._process.pid
            threads = self._process.num_threads()
            open_files_path = [str(x.path).replace("\\", "/") for x in self._process.open_files()]
            connections_uri = [f"{x.laddr}:{x.raddr} {str(x.status)}" for x in self._process.connections()]

            open_files = len(open_files_path)
            connections = len(connections_uri)
        except (psutil.NoSuchProcess, psutil.AccessDenied):
            process_id = os.getpid()
            threads = open_files = connections = 0
            open_files_path = []
            connections_uri = []

        return FootprintMetrics(
            start_time=self._footprint_start_time,
            uptime_seconds=uptime_seconds,
            uptime_formatted=uptime_formatted,
            memory_current=memory_current,
            memory_max=memory_max,
            memory_min=memory_min,
            memory_percent=memory_percent,
            cpu_percent_current=cpu_percent_current,
            cpu_percent_max=cpu_percent_max,
            cpu_percent_min=cpu_percent_min,
            cpu_percent_avg=cpu_percent_avg,
            cpu_time_seconds=cpu_time_seconds,
            disk_read_mb=disk_read_mb,
            disk_write_mb=disk_write_mb,
            disk_read_max=disk_read_max,
            disk_read_min=disk_read_min,
            disk_write_max=disk_write_max,
            disk_write_min=disk_write_min,
            network_sent_mb=network_sent_mb,
            network_recv_mb=network_recv_mb,
            network_sent_max=network_sent_max,
            network_sent_min=network_sent_min,
            network_recv_max=network_recv_max,
            network_recv_min=network_recv_min,
            process_id=process_id,
            threads=threads,
            open_files=open_files,
            connections=connections,
            open_files_path=open_files_path,
            connections_uri=connections_uri,
        )

    def print_footprint(self, detailed: bool = True) -> str:
        """
        Gibt den Footprint formatiert aus.

        Args:
            detailed: Wenn True, zeigt alle Details, sonst nur Zusammenfassung

        Returns:
            Formatierter Footprint-String
        """
        metrics = self.footprint()

        output = [
            "=" * 70,
            f"TOOLBOX FOOTPRINT - {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}",
            "=" * 70,
            f"\n📊 UPTIME",
            f"  Runtime: {metrics.uptime_formatted}",
            f"  Seconds: {metrics.uptime_seconds:.2f}s",
            f"\n💾 MEMORY USAGE",
            f"  Current:  {metrics.memory_current:.2f} MB ({metrics.memory_percent:.2f}%)",
            f"  Maximum:  {metrics.memory_max:.2f} MB",
            f"  Minimum:  {metrics.memory_min:.2f} MB",
        ]

        if detailed:
            helper_ = '\n\t- '.join(metrics.open_files_path)
            helper__ = '\n\t- '.join(metrics.connections_uri)
            output.extend([
                f"\n⚙️  CPU USAGE",
                f"  Current:  {metrics.cpu_percent_current:.2f}%",
                f"  Maximum:  {metrics.cpu_percent_max:.2f}%",
                f"  Minimum:  {metrics.cpu_percent_min:.2f}%",
                f"  Average:  {metrics.cpu_percent_avg:.2f}%",
                f"  CPU Time: {metrics.cpu_time_seconds:.2f}s",
                f"\n💿 DISK I/O",
                f"  Read:     {metrics.disk_read_mb:.2f} MB (Max: {metrics.disk_read_max:.2f}, Min: {metrics.disk_read_min:.2f})",
                f"  Write:    {metrics.disk_write_mb:.2f} MB (Max: {metrics.disk_write_max:.2f}, Min: {metrics.disk_write_min:.2f})",
                f"\n🌐 NETWORK I/O",
                f"  Sent:     {metrics.network_sent_mb:.2f} MB (Max: {metrics.network_sent_max:.2f}, Min: {metrics.network_sent_min:.2f})",
                f"  Received: {metrics.network_recv_mb:.2f} MB (Max: {metrics.network_recv_max:.2f}, Min: {metrics.network_recv_min:.2f})",
                f"\n🔧 PROCESS INFO",
                f"  PID:         {metrics.process_id}",
                f"  Threads:     {metrics.threads}",
                f"\n📂 OPEN FILES",
                f"  Open Files:  {metrics.open_files}",
                f"  Open Files Path: \n\t- {helper_}",
                f"\n🔗 NETWORK CONNECTIONS",
                f"  Connections: {metrics.connections}",
                f"  Connections URI: \n\t- {helper__}",
            ])

        output.append("=" * 70)

        return "\n".join(output)



    def start_server(self):
        from toolboxv2.utils.clis.api import manage_server
        if self.is_server:
            return
        manage_server("start")
        self.is_server = False

    @staticmethod
    def exit_main(*args, **kwargs):
        """proxi attr"""

    @staticmethod
    async def hide_console(*args, **kwargs):
        """proxi attr"""

    @staticmethod
    async def show_console(*args, **kwargs):
        """proxi attr"""

    @staticmethod
    async def disconnect(*args, **kwargs):
        """proxi attr"""

    def set_logger(self, debug=False):
        """proxi attr"""

    @property
    def debug(self):
        """proxi attr"""
        return self._debug

    def debug_rains(self, e):
        """proxi attr"""

    def set_flows(self, r):
        """proxi attr"""

    async def run_flows(self, name, **kwargs):
        """proxi attr"""

    def rrun_flows(self, name, **kwargs):
        """proxi attr"""

    def idle(self):
        import time
        self.print("idle")
        try:
            while self.alive:
                time.sleep(1)
        except KeyboardInterrupt:
            pass
        self.print("idle done")

    async def a_idle(self):
        self.print("a idle (running :"+("online)" if hasattr(self, 'daemon_app') else "offline)"))
        try:
            if hasattr(self, 'daemon_app'):
                await self.daemon_app.connect(self)
            else:
                while self.alive:
                    await asyncio.sleep(1)
        except KeyboardInterrupt:
            pass
        self.print("a idle done")

    @debug.setter
    def debug(self, value):
        """proxi attr"""

    def _coppy_mod(self, content, new_mod_dir, mod_name, file_type='py'):
        """proxi attr"""

    def _pre_lib_mod(self, mod_name, path_to="./runtime", file_type='py'):
        """proxi attr"""

    def _copy_load(self, mod_name, file_type='py', **kwargs):
        """proxi attr"""

    def inplace_load_instance(self, mod_name, loc="toolboxv2.mods.", spec='app', save=True):
        """proxi attr"""

    def save_instance(self, instance, modular_id, spec='app', instance_type="file/application", tools_class=None):
        """proxi attr"""

    def save_initialized_module(self, tools_class, spec):
        """proxi attr"""

    def mod_online(self, mod_name, installed=False):
        """proxi attr"""

    def _get_function(self,
                      name: Enum or None,
                      state: bool = True,
                      specification: str = "app",
                      metadata=False, as_str: tuple or None = None, r=0):
        """proxi attr"""

    def save_exit(self):
        """proxi attr"""

    def load_mod(self, mod_name: str, mlm='I', **kwargs):
        """proxi attr"""

    async def init_module(self, modular):
        return await self.load_mod(modular)

    async def load_external_mods(self):
        """proxi attr"""

    async def load_all_mods_in_file(self, working_dir="mods"):
        """proxi attr"""

    def get_all_mods(self, working_dir="mods", path_to="./runtime"):
        """proxi attr"""

    def remove_all_modules(self, delete=False):
        for mod in list(self.functions.keys()):
            self.logger.info(f"closing: {mod}")
            self.remove_mod(mod, delete=delete)

    async def a_remove_all_modules(self, delete=False):
        for mod in list(self.functions.keys()):
            self.logger.info(f"closing: {mod}")
            await self.a_remove_mod(mod, delete=delete)

    def print_ok(self):
        """proxi attr"""
        self.logger.info("OK")

    def reload_mod(self, mod_name, spec='app', is_file=True, loc="toolboxv2.mods."):
        """proxi attr"""

    def watch_mod(self, mod_name, spec='app', loc="toolboxv2.mods.", use_thread=True, path_name=None):
        """proxi attr"""

    def remove_mod(self, mod_name, spec='app', delete=True):
        """proxi attr"""

    async def a_remove_mod(self, mod_name, spec='app', delete=True):
        """proxi attr"""

    def exit(self):
        """proxi attr"""

    def web_context(self) -> str:
        """returns the build index ( toolbox web component )"""

    async def a_exit(self):
        """proxi attr"""

    def save_load(self, modname, spec='app'):
        """proxi attr"""

    def get_function(self, name: Enum or tuple, **kwargs):
        """
        Kwargs for _get_function
            metadata:: return the registered function dictionary
                stateless: (function_data, None), 0
                stateful: (function_data, higher_order_function), 0
            state::boolean
                specification::str default app
        """

    def run_a_from_sync(self, function, *args):
        """
        run a async fuction
        """

    def run_bg_task_advanced(self, task, *args, **kwargs):
        """
        proxi attr
        """

    def wait_for_bg_tasks(self, timeout=None):
        """
        proxi attr
        """

    def run_bg_task(self, task):
        """
                run a async fuction
                """
    def run_function(self, mod_function_name: Enum or tuple,
                     tb_run_function_with_state=True,
                     tb_run_with_specification='app',
                     args_=None,
                     kwargs_=None,
                     *args,
                     **kwargs) -> Result:

        """proxi attr"""

    async def a_run_function(self, mod_function_name: Enum or tuple,
                             tb_run_function_with_state=True,
                             tb_run_with_specification='app',
                             args_=None,
                             kwargs_=None,
                             *args,
                             **kwargs) -> Result:

        """proxi attr"""

    def fuction_runner(self, function, function_data: dict, args: list, kwargs: dict, t0=.0):
        """
        parameters = function_data.get('params')
        modular_name = function_data.get('module_name')
        function_name = function_data.get('func_name')
        mod_function_name = f"{modular_name}.{function_name}"

        proxi attr
        """

    async def a_fuction_runner(self, function, function_data: dict, args: list, kwargs: dict):
        """
        parameters = function_data.get('params')
        modular_name = function_data.get('module_name')
        function_name = function_data.get('func_name')
        mod_function_name = f"{modular_name}.{function_name}"

        proxi attr
        """

    async def run_http(self, mod_function_name: Enum or str or tuple, function_name=None, method="GET",
                       args_=None,
                       kwargs_=None,
                       *args, **kwargs):
        """run a function remote via http / https"""

    def run_any(self, mod_function_name: Enum or str or tuple, backwords_compability_variabel_string_holder=None,
                get_results=False, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None,
                kwargs_=None,
                *args, **kwargs):
        """proxi attr"""

    async def a_run_any(self, mod_function_name: Enum or str or tuple,
                        backwords_compability_variabel_string_holder=None,
                        get_results=False, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None,
                        kwargs_=None,
                        *args, **kwargs):
        """proxi attr"""

    def get_mod(self, name, spec='app') -> ModuleType or MainToolType:
        """proxi attr"""

    @staticmethod
    def print(text, *args, **kwargs):
        """proxi attr"""

    @staticmethod
    def sprint(text, *args, **kwargs):
        """proxi attr"""

    # ----------------------------------------------------------------
    # Decorators for the toolbox

    def _register_function(self, module_name, func_name, data):
        """proxi attr"""

    def _create_decorator(self, type_: str,
                          name: str = "",
                          mod_name: str = "",
                          level: int = -1,
                          restrict_in_virtual_mode: bool = False,
                          api: bool = False,
                          helper: str = "",
                          version: str or None = None,
                          initial=False,
                          exit_f=False,
                          test=True,
                          samples=None,
                          state=None,
                          pre_compute=None,
                          post_compute=None,
                          memory_cache=False,
                          file_cache=False,
                          row=False,
                          request_as_kwarg=False,
                          memory_cache_max_size=100,
                          memory_cache_ttl=300,
                          websocket_handler: str | None = None,):
        """proxi attr"""

        # data = {
        #     "type": type_,
        #     "module_name": module_name,
        #     "func_name": func_name,
        #     "level": level,
        #     "restrict_in_virtual_mode": restrict_in_virtual_mode,
        #     "func": func,
        #     "api": api,
        #     "helper": helper,
        #     "version": version,
        #     "initial": initial,
        #     "exit_f": exit_f,
        #     "__module__": func.__module__,
        #     "signature": sig,
        #     "params": params,
        #     "state": (
        #         False if len(params) == 0 else params[0] in ['self', 'state', 'app']) if state is None else state,
        #     "do_test": test,
        #     "samples": samples,
        #     "request_as_kwarg": request_as_kwarg,

    def tb(self, name=None,
           mod_name: str = "",
           helper: str = "",
           version: str or None = None,
           test: bool = True,
           restrict_in_virtual_mode: bool = False,
           api: bool = False,
           initial: bool = False,
           exit_f: bool = False,
           test_only: bool = False,
           memory_cache: bool = False,
           file_cache: bool = False,
           row=False,
           request_as_kwarg: bool = False,
           state: bool or None = None,
           level: int = 0,
           memory_cache_max_size: int = 100,
           memory_cache_ttl: int = 300,
           samples: list or dict or None = None,
           interface: ToolBoxInterfaces or None or str = None,
           pre_compute=None,
           post_compute=None,
           api_methods=None,
           websocket_handler: str | None = None,
           ):
        """
    A decorator for registering and configuring functions within a module.

    This decorator is used to wrap functions with additional functionality such as caching, API conversion, and lifecycle management (initialization and exit). It also handles the registration of the function in the module's function registry.

    Args:
        name (str, optional): The name to register the function under. Defaults to the function's own name.
        mod_name (str, optional): The name of the module the function belongs to.
        helper (str, optional): A helper string providing additional information about the function.
        version (str or None, optional): The version of the function or module.
        test (bool, optional): Flag to indicate if the function is for testing purposes.
        restrict_in_virtual_mode (bool, optional): Flag to restrict the function in virtual mode.
        api (bool, optional): Flag to indicate if the function is part of an API.
        initial (bool, optional): Flag to indicate if the function should be executed at initialization.
        exit_f (bool, optional): Flag to indicate if the function should be executed at exit.
        test_only (bool, optional): Flag to indicate if the function should only be used for testing.
        memory_cache (bool, optional): Flag to enable memory caching for the function.
        request_as_kwarg (bool, optional): Flag to get request if the fuction is calld from api.
        file_cache (bool, optional): Flag to enable file caching for the function.
        row (bool, optional): rather to auto wrap the result in Result type default False means no row data aka result type
        state (bool or None, optional): Flag to indicate if the function maintains state.
        level (int, optional): The level of the function, used for prioritization or categorization.
        memory_cache_max_size (int, optional): Maximum size of the memory cache.
        memory_cache_ttl (int, optional): Time-to-live for the memory cache entries.
        samples (list or dict or None, optional): Samples or examples of function usage.
        interface (str, optional): The interface type for the function.
        pre_compute (callable, optional): A function to be called before the main function.
        post_compute (callable, optional): A function to be called after the main function.
        api_methods (list[str], optional): default ["AUTO"] (GET if not params, POST if params) , GET, POST, PUT or DELETE.

    Returns:
        function: The decorated function with additional processing and registration capabilities.
    """
        if interface is None:
            interface = "tb"
        if test_only and 'test' not in self.id:
            return lambda *args, **kwargs: args
        return self._create_decorator(interface,
                                      name,
                                      mod_name,
                                      level=level,
                                      restrict_in_virtual_mode=restrict_in_virtual_mode,
                                      helper=helper,
                                      api=api,
                                      version=version,
                                      initial=initial,
                                      exit_f=exit_f,
                                      test=test,
                                      samples=samples,
                                      state=state,
                                      pre_compute=pre_compute,
                                      post_compute=post_compute,
                                      memory_cache=memory_cache,
                                      file_cache=file_cache,
                                      row=row,
                                      request_as_kwarg=request_as_kwarg,
                                      memory_cache_max_size=memory_cache_max_size,
                                      memory_cache_ttl=memory_cache_ttl)

    def print_functions(self, name=None):


        if not self.functions:
            return

        def helper(_functions):
            for func_name, data in _functions.items():
                if not isinstance(data, dict):
                    continue

                func_type = data.get('type', 'Unknown')
                func_level = 'r' if data['level'] == -1 else data['level']
                api_status = 'Api' if data.get('api', False) else 'Non-Api'

                print(f"  Function: {func_name}{data.get('signature', '()')}; "
                      f"Type: {func_type}, Level: {func_level}, {api_status}")

        if name is not None:
            functions = self.functions.get(name)
            if functions is not None:
                print(f"\nModule: {name}; Type: {functions.get('app_instance_type', 'Unknown')}")
                helper(functions)
                return
        for module, functions in self.functions.items():
            print(f"\nModule: {module}; Type: {functions.get('app_instance_type', 'Unknown')}")
            helper(functions)

    def save_autocompletion_dict(self):
        """proxi attr"""

    def get_autocompletion_dict(self):
        """proxi attr"""

    def get_username(self, get_input=False, default="loot") -> str:
        """proxi attr"""

    def save_registry_as_enums(self, directory: str, filename: str):
        """proxi attr"""

    async def execute_all_functions_(self, m_query='', f_query=''):
        print("Executing all functions")
        from ..extras import generate_test_cases
        all_data = {
            "modular_run": 0,
            "modular_fatal_error": 0,
            "errors": 0,
            "modular_sug": 0,
            "coverage": [],
            "total_coverage": {},
        }
        items = list(self.functions.items()).copy()
        for module_name, functions in items:
            infos = {
                "functions_run": 0,
                "functions_fatal_error": 0,
                "error": 0,
                "functions_sug": 0,
                'calls': {},
                'callse': {},
                "coverage": [0, 0],
            }
            all_data['modular_run'] += 1
            if not module_name.startswith(m_query):
                all_data['modular_sug'] += 1
                continue

            with Spinner(message=f"In {module_name}| "):
                f_items = list(functions.items()).copy()
                for function_name, function_data in f_items:
                    if not isinstance(function_data, dict):
                        continue
                    if not function_name.startswith(f_query):
                        continue
                    test: list = function_data.get('do_test')
                    # print(test, module_name, function_name, function_data)
                    infos["coverage"][0] += 1
                    if test is False:
                        continue

                    with Spinner(message=f"\t\t\t\t\t\tfuction {function_name}..."):
                        params: list = function_data.get('params')
                        sig: signature = function_data.get('signature')
                        state: bool = function_data.get('state')
                        samples: bool = function_data.get('samples')

                        test_kwargs_list = [{}]

                        if params is not None:
                            test_kwargs_list = samples if samples is not None else generate_test_cases(sig=sig)
                            # print(test_kwargs)
                            # print(test_kwargs[0])
                            # test_kwargs = test_kwargs_list[0]
                        # print(module_name, function_name, test_kwargs_list)
                        infos["coverage"][1] += 1
                        for test_kwargs in test_kwargs_list:
                            try:
                                # print(f"test Running {state=} |{module_name}.{function_name}")
                                result = await self.a_run_function((module_name, function_name),
                                                                   tb_run_function_with_state=state,
                                                                   **test_kwargs)
                                if not isinstance(result, Result):
                                    result = Result.ok(result)
                                if result.info.exec_code == 0:
                                    infos['calls'][function_name] = [test_kwargs, str(result)]
                                    infos['functions_sug'] += 1
                                else:
                                    infos['functions_sug'] += 1
                                    infos['error'] += 1
                                    infos['callse'][function_name] = [test_kwargs, str(result)]
                            except Exception as e:
                                infos['functions_fatal_error'] += 1
                                infos['callse'][function_name] = [test_kwargs, str(e)]
                            finally:
                                infos['functions_run'] += 1

                if infos['functions_run'] == infos['functions_sug']:
                    all_data['modular_sug'] += 1
                else:
                    all_data['modular_fatal_error'] += 1
                if infos['error'] > 0:
                    all_data['errors'] += infos['error']

                all_data[module_name] = infos
                if infos['coverage'][0] == 0:
                    c = 0
                else:
                    c = infos['coverage'][1] / infos['coverage'][0]
                all_data["coverage"].append(f"{module_name}:{c:.2f}\n")
        total_coverage = sum([float(t.split(":")[-1]) for t in all_data["coverage"]]) / len(all_data["coverage"])
        print(
            f"\n{all_data['modular_run']=}\n{all_data['modular_sug']=}\n{all_data['modular_fatal_error']=}\n{total_coverage=}")
        d = analyze_data(all_data)
        return Result.ok(data=all_data, data_info=d)

    async def execute_function_test(self, module_name: str, function_name: str,
                                    function_data: dict, test_kwargs: dict,
                                    profiler: cProfile.Profile) -> tuple[bool, str, dict, float]:
        start_time = time.time()
        with profile_section(profiler, hasattr(self, 'enable_profiling') and self.enable_profiling):
            try:
                result = await self.a_run_function(
                    (module_name, function_name),
                    tb_run_function_with_state=function_data.get('state'),
                    **test_kwargs
                )

                if not isinstance(result, Result):
                    result = Result.ok(result)

                success = result.info.exec_code == 0
                execution_time = time.time() - start_time
                return success, str(result), test_kwargs, execution_time
            except Exception as e:
                execution_time = time.time() - start_time
                return False, str(e), test_kwargs, execution_time

    async def process_function(self, module_name: str, function_name: str,
                               function_data: dict, profiler: cProfile.Profile) -> tuple[str, ModuleInfo]:
        start_time = time.time()
        info = ModuleInfo()

        with profile_section(profiler, hasattr(self, 'enable_profiling') and self.enable_profiling):
            if not isinstance(function_data, dict):
                return function_name, info

            test = function_data.get('do_test')
            info.coverage[0] += 1

            if test is False:
                return function_name, info

            params = function_data.get('params')
            sig = function_data.get('signature')
            samples = function_data.get('samples')

            test_kwargs_list = [{}] if params is None else (
                samples if samples is not None else generate_test_cases(sig=sig)
            )

            info.coverage[1] += 1

            # Create tasks for all test cases
            tasks = [
                self.execute_function_test(module_name, function_name, function_data, test_kwargs, profiler)
                for test_kwargs in test_kwargs_list
            ]

            # Execute all tests concurrently
            results = await asyncio.gather(*tasks)

            total_execution_time = 0
            for success, result_str, test_kwargs, execution_time in results:
                info.functions_run += 1
                total_execution_time += execution_time

                if success:
                    info.functions_sug += 1
                    info.calls[function_name] = [test_kwargs, result_str]
                else:
                    info.functions_sug += 1
                    info.error += 1
                    info.callse[function_name] = [test_kwargs, result_str]

            info.execution_time = time.time() - start_time
            return function_name, info

    async def process_module(self, module_name: str, functions: dict,
                             f_query: str, profiler: cProfile.Profile) -> tuple[str, ModuleInfo]:
        start_time = time.time()

        with profile_section(profiler, hasattr(self, 'enable_profiling') and self.enable_profiling):
            async with asyncio.Semaphore(mp.cpu_count()):
                tasks = [
                    self.process_function(module_name, fname, fdata, profiler)
                    for fname, fdata in functions.items()
                    if fname.startswith(f_query)
                ]

                if not tasks:
                    return module_name, ModuleInfo()

                results = await asyncio.gather(*tasks)

                # Combine results from all functions in the module
                combined_info = ModuleInfo()
                total_execution_time = 0

                for _, info in results:
                    combined_info.functions_run += info.functions_run
                    combined_info.functions_fatal_error += info.functions_fatal_error
                    combined_info.error += info.error
                    combined_info.functions_sug += info.functions_sug
                    combined_info.calls.update(info.calls)
                    combined_info.callse.update(info.callse)
                    combined_info.coverage[0] += info.coverage[0]
                    combined_info.coverage[1] += info.coverage[1]
                    total_execution_time += info.execution_time

                combined_info.execution_time = time.time() - start_time
                return module_name, combined_info

    async def execute_all_functions(self, m_query='', f_query='', enable_profiling=True):
        """
        Execute all functions with parallel processing and optional profiling.

        Args:
            m_query (str): Module name query filter
            f_query (str): Function name query filter
            enable_profiling (bool): Enable detailed profiling information
        """
        print("Executing all functions in parallel" + (" with profiling" if enable_profiling else ""))

        start_time = time.time()
        stats = ExecutionStats()
        items = list(self.functions.items()).copy()

        # Set up profiling
        self.enable_profiling = enable_profiling
        profiler = cProfile.Profile()

        with profile_section(profiler, enable_profiling):
            # Filter modules based on query
            filtered_modules = [
                (mname, mfuncs) for mname, mfuncs in items
                if mname.startswith(m_query)
            ]

            stats.modular_run = len(filtered_modules)

            # Process all modules concurrently
            async with asyncio.Semaphore(mp.cpu_count()):
                tasks = [
                    self.process_module(mname, mfuncs, f_query, profiler)
                    for mname, mfuncs in filtered_modules
                ]

                results = await asyncio.gather(*tasks)

            # Combine results and calculate statistics
            for module_name, info in results:
                if info.functions_run == info.functions_sug:
                    stats.modular_sug += 1
                else:
                    stats.modular_fatal_error += 1

                stats.errors += info.error

                # Calculate coverage
                coverage = (info.coverage[1] / info.coverage[0]) if info.coverage[0] > 0 else 0
                stats.coverage.append(f"{module_name}:{coverage:.2f}\n")

                # Store module info
                stats.__dict__[module_name] = info

            # Calculate total coverage
            total_coverage = (
                sum(float(t.split(":")[-1]) for t in stats.coverage) / len(stats.coverage)
                if stats.coverage else 0
            )

            stats.total_execution_time = time.time() - start_time

            # Generate profiling stats if enabled
            if enable_profiling:
                s = io.StringIO()
                ps = pstats.Stats(profiler, stream=s).sort_stats('cumulative')
                ps.print_stats()
                stats.profiling_data = {
                    'detailed_stats': s.getvalue(),
                    'total_time': stats.total_execution_time,
                    'function_count': stats.modular_run,
                    'successful_functions': stats.modular_sug
                }

            print(
                f"\n{stats.modular_run=}"
                f"\n{stats.modular_sug=}"
                f"\n{stats.modular_fatal_error=}"
                f"\n{total_coverage=}"
                f"\nTotal execution time: {stats.total_execution_time:.2f}s"
            )

            if enable_profiling:
                print("\nProfiling Summary:")
                print(f"{'=' * 50}")
                print("Top 10 time-consuming functions:")
                ps.print_stats(10)

            analyzed_data = analyze_data(stats.__dict__)
            return Result.ok(data=stats.__dict__, data_info=analyzed_data)

debug property writable

proxi attr

a_exit() async

proxi attr

Source code in toolboxv2/utils/system/types.py

async def a_exit(self):
    """proxi attr"""

a_fuction_runner(function, function_data, args, kwargs) async

parameters = function_data.get('params') modular_name = function_data.get('module_name') function_name = function_data.get('func_name') mod_function_name = f"{modular_name}.{function_name}"

proxi attr

Source code in toolboxv2/utils/system/types.py

async def a_fuction_runner(self, function, function_data: dict, args: list, kwargs: dict):
    """
    parameters = function_data.get('params')
    modular_name = function_data.get('module_name')
    function_name = function_data.get('func_name')
    mod_function_name = f"{modular_name}.{function_name}"

    proxi attr
    """

a_remove_mod(mod_name, spec='app', delete=True) async

proxi attr

Source code in toolboxv2/utils/system/types.py

async def a_remove_mod(self, mod_name, spec='app', delete=True):
    """proxi attr"""

a_run_any(mod_function_name, backwords_compability_variabel_string_holder=None, get_results=False, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None, kwargs_=None, *args, **kwargs) async

proxi attr

Source code in toolboxv2/utils/system/types.py

async def a_run_any(self, mod_function_name: Enum or str or tuple,
                    backwords_compability_variabel_string_holder=None,
                    get_results=False, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None,
                    kwargs_=None,
                    *args, **kwargs):
    """proxi attr"""

a_run_function(mod_function_name, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None, kwargs_=None, *args, **kwargs) async

proxi attr

Source code in toolboxv2/utils/system/types.py

async def a_run_function(self, mod_function_name: Enum or tuple,
                         tb_run_function_with_state=True,
                         tb_run_with_specification='app',
                         args_=None,
                         kwargs_=None,
                         *args,
                         **kwargs) -> Result:

    """proxi attr"""

debug_rains(e)

proxi attr

Source code in toolboxv2/utils/system/types.py

def debug_rains(self, e):
    """proxi attr"""

disconnect(*args, **kwargs) async staticmethod

proxi attr

Source code in toolboxv2/utils/system/types.py

@staticmethod
async def disconnect(*args, **kwargs):
    """proxi attr"""

execute_all_functions(m_query='', f_query='', enable_profiling=True) async

Execute all functions with parallel processing and optional profiling.

Parameters:

Name	Type	Description	Default
m_query	str	Module name query filter	''
f_query	str	Function name query filter	''
enable_profiling	bool	Enable detailed profiling information	True

Source code in toolboxv2/utils/system/types.py

async def execute_all_functions(self, m_query='', f_query='', enable_profiling=True):
    """
    Execute all functions with parallel processing and optional profiling.

    Args:
        m_query (str): Module name query filter
        f_query (str): Function name query filter
        enable_profiling (bool): Enable detailed profiling information
    """
    print("Executing all functions in parallel" + (" with profiling" if enable_profiling else ""))

    start_time = time.time()
    stats = ExecutionStats()
    items = list(self.functions.items()).copy()

    # Set up profiling
    self.enable_profiling = enable_profiling
    profiler = cProfile.Profile()

    with profile_section(profiler, enable_profiling):
        # Filter modules based on query
        filtered_modules = [
            (mname, mfuncs) for mname, mfuncs in items
            if mname.startswith(m_query)
        ]

        stats.modular_run = len(filtered_modules)

        # Process all modules concurrently
        async with asyncio.Semaphore(mp.cpu_count()):
            tasks = [
                self.process_module(mname, mfuncs, f_query, profiler)
                for mname, mfuncs in filtered_modules
            ]

            results = await asyncio.gather(*tasks)

        # Combine results and calculate statistics
        for module_name, info in results:
            if info.functions_run == info.functions_sug:
                stats.modular_sug += 1
            else:
                stats.modular_fatal_error += 1

            stats.errors += info.error

            # Calculate coverage
            coverage = (info.coverage[1] / info.coverage[0]) if info.coverage[0] > 0 else 0
            stats.coverage.append(f"{module_name}:{coverage:.2f}\n")

            # Store module info
            stats.__dict__[module_name] = info

        # Calculate total coverage
        total_coverage = (
            sum(float(t.split(":")[-1]) for t in stats.coverage) / len(stats.coverage)
            if stats.coverage else 0
        )

        stats.total_execution_time = time.time() - start_time

        # Generate profiling stats if enabled
        if enable_profiling:
            s = io.StringIO()
            ps = pstats.Stats(profiler, stream=s).sort_stats('cumulative')
            ps.print_stats()
            stats.profiling_data = {
                'detailed_stats': s.getvalue(),
                'total_time': stats.total_execution_time,
                'function_count': stats.modular_run,
                'successful_functions': stats.modular_sug
            }

        print(
            f"\n{stats.modular_run=}"
            f"\n{stats.modular_sug=}"
            f"\n{stats.modular_fatal_error=}"
            f"\n{total_coverage=}"
            f"\nTotal execution time: {stats.total_execution_time:.2f}s"
        )

        if enable_profiling:
            print("\nProfiling Summary:")
            print(f"{'=' * 50}")
            print("Top 10 time-consuming functions:")
            ps.print_stats(10)

        analyzed_data = analyze_data(stats.__dict__)
        return Result.ok(data=stats.__dict__, data_info=analyzed_data)

exit()

proxi attr

Source code in toolboxv2/utils/system/types.py

def exit(self):
    """proxi attr"""

exit_main(*args, **kwargs) staticmethod

proxi attr

Source code in toolboxv2/utils/system/types.py

@staticmethod
def exit_main(*args, **kwargs):
    """proxi attr"""

footprint(update_tracking=True)

Erfasst den aktuellen Ressourcen-Footprint der Toolbox-Instanz.

Parameters:

Name	Type	Description	Default
update_tracking	bool	Wenn True, aktualisiert Min/Max/Avg-Tracking	True

Returns:

Type	Description
FootprintMetrics	FootprintMetrics mit allen erfassten Metriken

Source code in toolboxv2/utils/system/types.py

def footprint(self, update_tracking: bool = True) -> FootprintMetrics:
    """
    Erfasst den aktuellen Ressourcen-Footprint der Toolbox-Instanz.

    Args:
        update_tracking: Wenn True, aktualisiert Min/Max/Avg-Tracking

    Returns:
        FootprintMetrics mit allen erfassten Metriken
    """
    current_time = time.time()
    uptime_seconds = current_time - self._footprint_start_time

    # Formatierte Uptime
    uptime_delta = timedelta(seconds=int(uptime_seconds))
    uptime_formatted = str(uptime_delta)

    # Memory Metrics (in MB)
    try:
        mem_info = self._process.memory_info()
        memory_current = mem_info.rss / (1024 * 1024)  # Bytes zu MB
        memory_percent = self._process.memory_percent()

        if update_tracking:
            self._update_metric_tracking('memory', memory_current)

        memory_max = self._footprint_metrics['memory']['max']
        memory_min = self._footprint_metrics['memory']['min']
        if memory_min == float('inf'):
            memory_min = memory_current
    except (psutil.NoSuchProcess, psutil.AccessDenied):
        memory_current = memory_max = memory_min = memory_percent = 0

    # CPU Metrics
    try:
        cpu_percent_current = self._process.cpu_percent(interval=0.1)
        cpu_times = self._process.cpu_times()
        cpu_time_seconds = cpu_times.user + cpu_times.system

        if update_tracking:
            self._update_metric_tracking('cpu', cpu_percent_current)

        cpu_percent_max = self._footprint_metrics['cpu']['max']
        cpu_percent_min = self._footprint_metrics['cpu']['min']
        cpu_percent_avg = self._get_metric_avg('cpu')

        if cpu_percent_min == float('inf'):
            cpu_percent_min = cpu_percent_current
    except (psutil.NoSuchProcess, psutil.AccessDenied):
        cpu_percent_current = cpu_percent_max = 0
        cpu_percent_min = cpu_percent_avg = cpu_time_seconds = 0

    # Disk I/O Metrics (in MB)
    try:
        io_counters = self._process.io_counters()
        disk_read_bytes = io_counters.read_bytes - self._initial_disk_read
        disk_write_bytes = io_counters.write_bytes - self._initial_disk_write

        disk_read_mb = disk_read_bytes / (1024 * 1024)
        disk_write_mb = disk_write_bytes / (1024 * 1024)

        if update_tracking:
            self._update_metric_tracking('disk_read', disk_read_mb)
            self._update_metric_tracking('disk_write', disk_write_mb)

        disk_read_max = self._footprint_metrics['disk_read']['max']
        disk_read_min = self._footprint_metrics['disk_read']['min']
        disk_write_max = self._footprint_metrics['disk_write']['max']
        disk_write_min = self._footprint_metrics['disk_write']['min']

        if disk_read_min == float('inf'):
            disk_read_min = disk_read_mb
        if disk_write_min == float('inf'):
            disk_write_min = disk_write_mb
    except (AttributeError, OSError, psutil.NoSuchProcess, psutil.AccessDenied):
        disk_read_mb = disk_write_mb = 0
        disk_read_max = disk_read_min = disk_write_max = disk_write_min = 0

    # Network I/O Metrics (in MB)
    try:
        net_io = psutil.net_io_counters()
        network_sent_bytes = net_io.bytes_sent - self._initial_network_sent
        network_recv_bytes = net_io.bytes_recv - self._initial_network_recv

        network_sent_mb = network_sent_bytes / (1024 * 1024)
        network_recv_mb = network_recv_bytes / (1024 * 1024)

        if update_tracking:
            self._update_metric_tracking('network_sent', network_sent_mb)
            self._update_metric_tracking('network_recv', network_recv_mb)

        network_sent_max = self._footprint_metrics['network_sent']['max']
        network_sent_min = self._footprint_metrics['network_sent']['min']
        network_recv_max = self._footprint_metrics['network_recv']['max']
        network_recv_min = self._footprint_metrics['network_recv']['min']

        if network_sent_min == float('inf'):
            network_sent_min = network_sent_mb
        if network_recv_min == float('inf'):
            network_recv_min = network_recv_mb
    except (AttributeError, OSError):
        network_sent_mb = network_recv_mb = 0
        network_sent_max = network_sent_min = 0
        network_recv_max = network_recv_min = 0

    # Process Info
    try:
        process_id = self._process.pid
        threads = self._process.num_threads()
        open_files_path = [str(x.path).replace("\\", "/") for x in self._process.open_files()]
        connections_uri = [f"{x.laddr}:{x.raddr} {str(x.status)}" for x in self._process.connections()]

        open_files = len(open_files_path)
        connections = len(connections_uri)
    except (psutil.NoSuchProcess, psutil.AccessDenied):
        process_id = os.getpid()
        threads = open_files = connections = 0
        open_files_path = []
        connections_uri = []

    return FootprintMetrics(
        start_time=self._footprint_start_time,
        uptime_seconds=uptime_seconds,
        uptime_formatted=uptime_formatted,
        memory_current=memory_current,
        memory_max=memory_max,
        memory_min=memory_min,
        memory_percent=memory_percent,
        cpu_percent_current=cpu_percent_current,
        cpu_percent_max=cpu_percent_max,
        cpu_percent_min=cpu_percent_min,
        cpu_percent_avg=cpu_percent_avg,
        cpu_time_seconds=cpu_time_seconds,
        disk_read_mb=disk_read_mb,
        disk_write_mb=disk_write_mb,
        disk_read_max=disk_read_max,
        disk_read_min=disk_read_min,
        disk_write_max=disk_write_max,
        disk_write_min=disk_write_min,
        network_sent_mb=network_sent_mb,
        network_recv_mb=network_recv_mb,
        network_sent_max=network_sent_max,
        network_sent_min=network_sent_min,
        network_recv_max=network_recv_max,
        network_recv_min=network_recv_min,
        process_id=process_id,
        threads=threads,
        open_files=open_files,
        connections=connections,
        open_files_path=open_files_path,
        connections_uri=connections_uri,
    )

fuction_runner(function, function_data, args, kwargs, t0=0.0)

parameters = function_data.get('params') modular_name = function_data.get('module_name') function_name = function_data.get('func_name') mod_function_name = f"{modular_name}.{function_name}"

proxi attr

Source code in toolboxv2/utils/system/types.py

def fuction_runner(self, function, function_data: dict, args: list, kwargs: dict, t0=.0):
    """
    parameters = function_data.get('params')
    modular_name = function_data.get('module_name')
    function_name = function_data.get('func_name')
    mod_function_name = f"{modular_name}.{function_name}"

    proxi attr
    """

get_all_mods(working_dir='mods', path_to='./runtime')

proxi attr

Source code in toolboxv2/utils/system/types.py

def get_all_mods(self, working_dir="mods", path_to="./runtime"):
    """proxi attr"""

get_autocompletion_dict()

proxi attr

Source code in toolboxv2/utils/system/types.py

def get_autocompletion_dict(self):
    """proxi attr"""

get_function(name, **kwargs)

Kwargs for _get_function metadata:: return the registered function dictionary stateless: (function_data, None), 0 stateful: (function_data, higher_order_function), 0 state::boolean specification::str default app

Source code in toolboxv2/utils/system/types.py

def get_function(self, name: Enum or tuple, **kwargs):
    """
    Kwargs for _get_function
        metadata:: return the registered function dictionary
            stateless: (function_data, None), 0
            stateful: (function_data, higher_order_function), 0
        state::boolean
            specification::str default app
    """

get_mod(name, spec='app')

proxi attr

Source code in toolboxv2/utils/system/types.py

def get_mod(self, name, spec='app') -> ModuleType or MainToolType:
    """proxi attr"""

get_username(get_input=False, default='loot')

proxi attr

Source code in toolboxv2/utils/system/types.py

def get_username(self, get_input=False, default="loot") -> str:
    """proxi attr"""

hide_console(*args, **kwargs) async staticmethod

proxi attr

Source code in toolboxv2/utils/system/types.py

@staticmethod
async def hide_console(*args, **kwargs):
    """proxi attr"""

inplace_load_instance(mod_name, loc='toolboxv2.mods.', spec='app', save=True)

proxi attr

Source code in toolboxv2/utils/system/types.py

def inplace_load_instance(self, mod_name, loc="toolboxv2.mods.", spec='app', save=True):
    """proxi attr"""

load_all_mods_in_file(working_dir='mods') async

proxi attr

Source code in toolboxv2/utils/system/types.py

async def load_all_mods_in_file(self, working_dir="mods"):
    """proxi attr"""

load_external_mods() async

proxi attr

Source code in toolboxv2/utils/system/types.py

async def load_external_mods(self):
    """proxi attr"""

load_mod(mod_name, mlm='I', **kwargs)

proxi attr

Source code in toolboxv2/utils/system/types.py

def load_mod(self, mod_name: str, mlm='I', **kwargs):
    """proxi attr"""

mod_online(mod_name, installed=False)

proxi attr

Source code in toolboxv2/utils/system/types.py

def mod_online(self, mod_name, installed=False):
    """proxi attr"""

print(text, *args, **kwargs) staticmethod

proxi attr

Source code in toolboxv2/utils/system/types.py

@staticmethod
def print(text, *args, **kwargs):
    """proxi attr"""

print_footprint(detailed=True)

Gibt den Footprint formatiert aus.

Parameters:

Name	Type	Description	Default
detailed	bool	Wenn True, zeigt alle Details, sonst nur Zusammenfassung	True

Returns:

Type	Description
str	Formatierter Footprint-String

Source code in toolboxv2/utils/system/types.py

def print_footprint(self, detailed: bool = True) -> str:
    """
    Gibt den Footprint formatiert aus.

    Args:
        detailed: Wenn True, zeigt alle Details, sonst nur Zusammenfassung

    Returns:
        Formatierter Footprint-String
    """
    metrics = self.footprint()

    output = [
        "=" * 70,
        f"TOOLBOX FOOTPRINT - {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}",
        "=" * 70,
        f"\n📊 UPTIME",
        f"  Runtime: {metrics.uptime_formatted}",
        f"  Seconds: {metrics.uptime_seconds:.2f}s",
        f"\n💾 MEMORY USAGE",
        f"  Current:  {metrics.memory_current:.2f} MB ({metrics.memory_percent:.2f}%)",
        f"  Maximum:  {metrics.memory_max:.2f} MB",
        f"  Minimum:  {metrics.memory_min:.2f} MB",
    ]

    if detailed:
        helper_ = '\n\t- '.join(metrics.open_files_path)
        helper__ = '\n\t- '.join(metrics.connections_uri)
        output.extend([
            f"\n⚙️  CPU USAGE",
            f"  Current:  {metrics.cpu_percent_current:.2f}%",
            f"  Maximum:  {metrics.cpu_percent_max:.2f}%",
            f"  Minimum:  {metrics.cpu_percent_min:.2f}%",
            f"  Average:  {metrics.cpu_percent_avg:.2f}%",
            f"  CPU Time: {metrics.cpu_time_seconds:.2f}s",
            f"\n💿 DISK I/O",
            f"  Read:     {metrics.disk_read_mb:.2f} MB (Max: {metrics.disk_read_max:.2f}, Min: {metrics.disk_read_min:.2f})",
            f"  Write:    {metrics.disk_write_mb:.2f} MB (Max: {metrics.disk_write_max:.2f}, Min: {metrics.disk_write_min:.2f})",
            f"\n🌐 NETWORK I/O",
            f"  Sent:     {metrics.network_sent_mb:.2f} MB (Max: {metrics.network_sent_max:.2f}, Min: {metrics.network_sent_min:.2f})",
            f"  Received: {metrics.network_recv_mb:.2f} MB (Max: {metrics.network_recv_max:.2f}, Min: {metrics.network_recv_min:.2f})",
            f"\n🔧 PROCESS INFO",
            f"  PID:         {metrics.process_id}",
            f"  Threads:     {metrics.threads}",
            f"\n📂 OPEN FILES",
            f"  Open Files:  {metrics.open_files}",
            f"  Open Files Path: \n\t- {helper_}",
            f"\n🔗 NETWORK CONNECTIONS",
            f"  Connections: {metrics.connections}",
            f"  Connections URI: \n\t- {helper__}",
        ])

    output.append("=" * 70)

    return "\n".join(output)

print_ok()

proxi attr

Source code in toolboxv2/utils/system/types.py

def print_ok(self):
    """proxi attr"""
    self.logger.info("OK")

reload_mod(mod_name, spec='app', is_file=True, loc='toolboxv2.mods.')

proxi attr

Source code in toolboxv2/utils/system/types.py

def reload_mod(self, mod_name, spec='app', is_file=True, loc="toolboxv2.mods."):
    """proxi attr"""

remove_mod(mod_name, spec='app', delete=True)

proxi attr

Source code in toolboxv2/utils/system/types.py

def remove_mod(self, mod_name, spec='app', delete=True):
    """proxi attr"""

rrun_flows(name, **kwargs)

proxi attr

Source code in toolboxv2/utils/system/types.py

def rrun_flows(self, name, **kwargs):
    """proxi attr"""

run_a_from_sync(function, *args)

run a async fuction

Source code in toolboxv2/utils/system/types.py

def run_a_from_sync(self, function, *args):
    """
    run a async fuction
    """

run_any(mod_function_name, backwords_compability_variabel_string_holder=None, get_results=False, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None, kwargs_=None, *args, **kwargs)

proxi attr

Source code in toolboxv2/utils/system/types.py

def run_any(self, mod_function_name: Enum or str or tuple, backwords_compability_variabel_string_holder=None,
            get_results=False, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None,
            kwargs_=None,
            *args, **kwargs):
    """proxi attr"""

run_bg_task(task)

run a async fuction

Source code in toolboxv2/utils/system/types.py

def run_bg_task(self, task):
    """
            run a async fuction
            """

run_bg_task_advanced(task, *args, **kwargs)

proxi attr

Source code in toolboxv2/utils/system/types.py

def run_bg_task_advanced(self, task, *args, **kwargs):
    """
    proxi attr
    """

run_flows(name, **kwargs) async

proxi attr

Source code in toolboxv2/utils/system/types.py

async def run_flows(self, name, **kwargs):
    """proxi attr"""

run_function(mod_function_name, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None, kwargs_=None, *args, **kwargs)

proxi attr

Source code in toolboxv2/utils/system/types.py

def run_function(self, mod_function_name: Enum or tuple,
                 tb_run_function_with_state=True,
                 tb_run_with_specification='app',
                 args_=None,
                 kwargs_=None,
                 *args,
                 **kwargs) -> Result:

    """proxi attr"""

run_http(mod_function_name, function_name=None, method='GET', args_=None, kwargs_=None, *args, **kwargs) async

run a function remote via http / https

Source code in toolboxv2/utils/system/types.py

async def run_http(self, mod_function_name: Enum or str or tuple, function_name=None, method="GET",
                   args_=None,
                   kwargs_=None,
                   *args, **kwargs):
    """run a function remote via http / https"""

save_autocompletion_dict()

proxi attr

Source code in toolboxv2/utils/system/types.py

def save_autocompletion_dict(self):
    """proxi attr"""

save_exit()

proxi attr

Source code in toolboxv2/utils/system/types.py

def save_exit(self):
    """proxi attr"""

save_initialized_module(tools_class, spec)

proxi attr

Source code in toolboxv2/utils/system/types.py

def save_initialized_module(self, tools_class, spec):
    """proxi attr"""

save_instance(instance, modular_id, spec='app', instance_type='file/application', tools_class=None)

proxi attr

Source code in toolboxv2/utils/system/types.py

def save_instance(self, instance, modular_id, spec='app', instance_type="file/application", tools_class=None):
    """proxi attr"""

save_load(modname, spec='app')

proxi attr

Source code in toolboxv2/utils/system/types.py

def save_load(self, modname, spec='app'):
    """proxi attr"""

save_registry_as_enums(directory, filename)

proxi attr

Source code in toolboxv2/utils/system/types.py

def save_registry_as_enums(self, directory: str, filename: str):
    """proxi attr"""

set_flows(r)

proxi attr

Source code in toolboxv2/utils/system/types.py

def set_flows(self, r):
    """proxi attr"""

set_logger(debug=False)

proxi attr

Source code in toolboxv2/utils/system/types.py

def set_logger(self, debug=False):
    """proxi attr"""

show_console(*args, **kwargs) async staticmethod

proxi attr

Source code in toolboxv2/utils/system/types.py

@staticmethod
async def show_console(*args, **kwargs):
    """proxi attr"""

sprint(text, *args, **kwargs) staticmethod

proxi attr

Source code in toolboxv2/utils/system/types.py

@staticmethod
def sprint(text, *args, **kwargs):
    """proxi attr"""

tb(name=None, mod_name='', helper='', version=None, test=True, restrict_in_virtual_mode=False, api=False, initial=False, exit_f=False, test_only=False, memory_cache=False, file_cache=False, row=False, request_as_kwarg=False, state=None, level=0, memory_cache_max_size=100, memory_cache_ttl=300, samples=None, interface=None, pre_compute=None, post_compute=None, api_methods=None, websocket_handler=None)

A decorator for registering and configuring functions within a module.

This decorator is used to wrap functions with additional functionality such as caching, API conversion, and lifecycle management (initialization and exit). It also handles the registration of the function in the module's function registry.

Parameters:

Name	Type	Description	Default
name	str	The name to register the function under. Defaults to the function's own name.	None
mod_name	str	The name of the module the function belongs to.	''
helper	str	A helper string providing additional information about the function.	''
version	str or None	The version of the function or module.	None
test	bool	Flag to indicate if the function is for testing purposes.	True
restrict_in_virtual_mode	bool	Flag to restrict the function in virtual mode.	False
api	bool	Flag to indicate if the function is part of an API.	False
initial	bool	Flag to indicate if the function should be executed at initialization.	False
exit_f	bool	Flag to indicate if the function should be executed at exit.	False
test_only	bool	Flag to indicate if the function should only be used for testing.	False
memory_cache	bool	Flag to enable memory caching for the function.	False
request_as_kwarg	bool	Flag to get request if the fuction is calld from api.	False
file_cache	bool	Flag to enable file caching for the function.	False
row	bool	rather to auto wrap the result in Result type default False means no row data aka result type	False
state	bool or None	Flag to indicate if the function maintains state.	None
level	int	The level of the function, used for prioritization or categorization.	0
memory_cache_max_size	int	Maximum size of the memory cache.	100
memory_cache_ttl	int	Time-to-live for the memory cache entries.	300
samples	list or dict or None	Samples or examples of function usage.	None
interface	str	The interface type for the function.	None
pre_compute	callable	A function to be called before the main function.	None
post_compute	callable	A function to be called after the main function.	None
api_methods	list[str]	default ["AUTO"] (GET if not params, POST if params) , GET, POST, PUT or DELETE.	None

Returns:

Name	Type	Description
function		The decorated function with additional processing and registration capabilities.

Source code in toolboxv2/utils/system/types.py

def tb(self, name=None,
       mod_name: str = "",
       helper: str = "",
       version: str or None = None,
       test: bool = True,
       restrict_in_virtual_mode: bool = False,
       api: bool = False,
       initial: bool = False,
       exit_f: bool = False,
       test_only: bool = False,
       memory_cache: bool = False,
       file_cache: bool = False,
       row=False,
       request_as_kwarg: bool = False,
       state: bool or None = None,
       level: int = 0,
       memory_cache_max_size: int = 100,
       memory_cache_ttl: int = 300,
       samples: list or dict or None = None,
       interface: ToolBoxInterfaces or None or str = None,
       pre_compute=None,
       post_compute=None,
       api_methods=None,
       websocket_handler: str | None = None,
       ):
    """
A decorator for registering and configuring functions within a module.

This decorator is used to wrap functions with additional functionality such as caching, API conversion, and lifecycle management (initialization and exit). It also handles the registration of the function in the module's function registry.

Args:
    name (str, optional): The name to register the function under. Defaults to the function's own name.
    mod_name (str, optional): The name of the module the function belongs to.
    helper (str, optional): A helper string providing additional information about the function.
    version (str or None, optional): The version of the function or module.
    test (bool, optional): Flag to indicate if the function is for testing purposes.
    restrict_in_virtual_mode (bool, optional): Flag to restrict the function in virtual mode.
    api (bool, optional): Flag to indicate if the function is part of an API.
    initial (bool, optional): Flag to indicate if the function should be executed at initialization.
    exit_f (bool, optional): Flag to indicate if the function should be executed at exit.
    test_only (bool, optional): Flag to indicate if the function should only be used for testing.
    memory_cache (bool, optional): Flag to enable memory caching for the function.
    request_as_kwarg (bool, optional): Flag to get request if the fuction is calld from api.
    file_cache (bool, optional): Flag to enable file caching for the function.
    row (bool, optional): rather to auto wrap the result in Result type default False means no row data aka result type
    state (bool or None, optional): Flag to indicate if the function maintains state.
    level (int, optional): The level of the function, used for prioritization or categorization.
    memory_cache_max_size (int, optional): Maximum size of the memory cache.
    memory_cache_ttl (int, optional): Time-to-live for the memory cache entries.
    samples (list or dict or None, optional): Samples or examples of function usage.
    interface (str, optional): The interface type for the function.
    pre_compute (callable, optional): A function to be called before the main function.
    post_compute (callable, optional): A function to be called after the main function.
    api_methods (list[str], optional): default ["AUTO"] (GET if not params, POST if params) , GET, POST, PUT or DELETE.

Returns:
    function: The decorated function with additional processing and registration capabilities.
"""
    if interface is None:
        interface = "tb"
    if test_only and 'test' not in self.id:
        return lambda *args, **kwargs: args
    return self._create_decorator(interface,
                                  name,
                                  mod_name,
                                  level=level,
                                  restrict_in_virtual_mode=restrict_in_virtual_mode,
                                  helper=helper,
                                  api=api,
                                  version=version,
                                  initial=initial,
                                  exit_f=exit_f,
                                  test=test,
                                  samples=samples,
                                  state=state,
                                  pre_compute=pre_compute,
                                  post_compute=post_compute,
                                  memory_cache=memory_cache,
                                  file_cache=file_cache,
                                  row=row,
                                  request_as_kwarg=request_as_kwarg,
                                  memory_cache_max_size=memory_cache_max_size,
                                  memory_cache_ttl=memory_cache_ttl)

wait_for_bg_tasks(timeout=None)

proxi attr

Source code in toolboxv2/utils/system/types.py

def wait_for_bg_tasks(self, timeout=None):
    """
    proxi attr
    """

watch_mod(mod_name, spec='app', loc='toolboxv2.mods.', use_thread=True, path_name=None)

proxi attr

Source code in toolboxv2/utils/system/types.py

def watch_mod(self, mod_name, spec='app', loc="toolboxv2.mods.", use_thread=True, path_name=None):
    """proxi attr"""

web_context()

returns the build index ( toolbox web component )

Source code in toolboxv2/utils/system/types.py

def web_context(self) -> str:
    """returns the build index ( toolbox web component )"""

toolboxv2.MainTool

Source code in toolboxv2/utils/system/main_tool.py

class MainTool:
    toolID: str = ""
    # app = None
    interface = None
    spec = "app"
    name = ""
    color = "Bold"
    stuf = False

    def __init__(self, *args, **kwargs):
        """
        Standard constructor used for arguments pass
        Do not override. Use __ainit__ instead
        """
        self.__storedargs = args, kwargs
        self.tools = kwargs.get("tool", {})
        self.logger = kwargs.get("logs", get_logger())
        self.color = kwargs.get("color", "WHITE")
        self.todo = kwargs.get("load", kwargs.get("on_start", lambda: None))
        if "on_exit" in kwargs and isinstance(kwargs.get("on_exit"), Callable):
            self.on_exit =self.app.tb(
                mod_name=self.name,
                name=kwargs.get("on_exit").__name__,
                version=self.version if hasattr(self, 'version') else "0.0.0",
            )(kwargs.get("on_exit"))
        self.async_initialized = False
        if self.todo:
            try:
                if inspect.iscoroutinefunction(self.todo):
                    pass
                else:
                    self.todo()
                get_logger().info(f"{self.name} on load suspended")
            except Exception as e:
                get_logger().error(f"Error loading mod {self.name} {e}")
                if self.app.debug:
                    import traceback
                    traceback.print_exc()
        else:
            get_logger().info(f"{self.name} no load require")

    async def __ainit__(self, *args, **kwargs):
        self.version = kwargs.get("v", kwargs.get("version", "0.0.0"))
        self.tools = kwargs.get("tool", {})
        self.name = kwargs["name"]
        self.logger = kwargs.get("logs", get_logger())
        self.color = kwargs.get("color", "WHITE")
        self.todo = kwargs.get("load", kwargs.get("on_start"))
        if not hasattr(self, 'config'):
            self.config = {}
        self.user = None
        self.description = "A toolbox mod" if kwargs.get("description") is None else kwargs.get("description")
        if MainTool.interface is None:
            MainTool.interface = self.app.interface_type
        # Result.default(self.app.interface)

        if self.todo:
            try:
                if inspect.iscoroutinefunction(self.todo):
                    await self.todo()
                else:
                    pass
                await asyncio.sleep(0.1)
                get_logger().info(f"{self.name} on load suspended")
            except Exception as e:
                get_logger().error(f"Error loading mod {self.name} {e}")
                if self.app.debug:
                    import traceback
                    traceback.print_exc()
        else:
            get_logger().info(f"{self.name} no load require")
        self.app.print(f"TOOL : {self.spec}.{self.name} online")



    @property
    def app(self):
        return get_app(
            from_=f"{self.spec}.{self.name}|{self.toolID if self.toolID else '*' + MainTool.toolID} {self.interface if self.interface else MainTool.interface}")

    @app.setter
    def app(self, v):
        raise PermissionError(f"You cannot set the App Instance! {v=}")

    @staticmethod
    def return_result(error: ToolBoxError = ToolBoxError.none,
                      exec_code: int = 0,
                      help_text: str = "",
                      data_info=None,
                      data=None,
                      data_to=None):

        if data_to is None:
            data_to = MainTool.interface if MainTool.interface is not None else ToolBoxInterfaces.cli

        if data is None:
            data = {}

        if data_info is None:
            data_info = {}

        return Result(
            error,
            ToolBoxResult(data_info=data_info, data=data, data_to=data_to),
            ToolBoxInfo(exec_code=exec_code, help_text=help_text)
        )

    def print(self, message, end="\n", **kwargs):
        if self.stuf:
            return

        self.app.print(Style.style_dic[self.color] + self.name + Style.style_dic["END"] + ":", message, end=end,
                       **kwargs)

    def add_str_to_config(self, command):
        if len(command) != 2:
            self.logger.error('Invalid command must be key value')
            return False
        self.config[command[0]] = command[1]

    def webInstall(self, user_instance, construct_render) -> str:
        """"Returns a web installer for the given user instance and construct render template"""

    def get_version(self) -> str:
        """"Returns the version"""
        return self.version

    async def get_user(self, username: str) -> Result:
        return await self.app.a_run_any(CLOUDM_AUTHMANAGER.GET_USER_BY_NAME, username=username, get_results=True)

    async def __initobj(self):
        """Crutch used for __await__ after spawning"""
        assert not self.async_initialized
        self.async_initialized = True
        # pass the parameters to __ainit__ that passed to __init__
        await self.__ainit__(*self.__storedargs[0], **self.__storedargs[1])
        return self

    def __await__(self):
        return self.__initobj().__await__()

__init__(*args, **kwargs)

Standard constructor used for arguments pass Do not override. Use ainit instead

Source code in toolboxv2/utils/system/main_tool.py

def __init__(self, *args, **kwargs):
    """
    Standard constructor used for arguments pass
    Do not override. Use __ainit__ instead
    """
    self.__storedargs = args, kwargs
    self.tools = kwargs.get("tool", {})
    self.logger = kwargs.get("logs", get_logger())
    self.color = kwargs.get("color", "WHITE")
    self.todo = kwargs.get("load", kwargs.get("on_start", lambda: None))
    if "on_exit" in kwargs and isinstance(kwargs.get("on_exit"), Callable):
        self.on_exit =self.app.tb(
            mod_name=self.name,
            name=kwargs.get("on_exit").__name__,
            version=self.version if hasattr(self, 'version') else "0.0.0",
        )(kwargs.get("on_exit"))
    self.async_initialized = False
    if self.todo:
        try:
            if inspect.iscoroutinefunction(self.todo):
                pass
            else:
                self.todo()
            get_logger().info(f"{self.name} on load suspended")
        except Exception as e:
            get_logger().error(f"Error loading mod {self.name} {e}")
            if self.app.debug:
                import traceback
                traceback.print_exc()
    else:
        get_logger().info(f"{self.name} no load require")

__initobj() async

Crutch used for await after spawning

Source code in toolboxv2/utils/system/main_tool.py

async def __initobj(self):
    """Crutch used for __await__ after spawning"""
    assert not self.async_initialized
    self.async_initialized = True
    # pass the parameters to __ainit__ that passed to __init__
    await self.__ainit__(*self.__storedargs[0], **self.__storedargs[1])
    return self

get_version()

"Returns the version

Source code in toolboxv2/utils/system/main_tool.py

def get_version(self) -> str:
    """"Returns the version"""
    return self.version

webInstall(user_instance, construct_render)

"Returns a web installer for the given user instance and construct render template

Source code in toolboxv2/utils/system/main_tool.py

def webInstall(self, user_instance, construct_render) -> str:
    """"Returns a web installer for the given user instance and construct render template"""

toolboxv2.get_app(from_=None, name=None, args=AppArgs().default(), app_con=None, sync=False)

Source code in toolboxv2/utils/system/getting_and_closing_app.py

def get_app(from_=None, name=None, args=AppArgs().default(), app_con=None, sync=False) -> AppType:
    global registered_apps
    # name = None
    # inspect caller
    # from inspect import getouterframes, currentframe
    # print(f"get app requested from: {getouterframes(currentframe(), 2)[1].filename}::{getouterframes(currentframe(), 2)[1].lineno}")

    # print(f"get app requested from: {from_} withe name: {name}")
    logger = get_logger()
    logger.info(Style.GREYBG(f"get app requested from: {from_}"))
    if registered_apps[0] is not None:
        return registered_apps[0]

    if app_con is None:
        from ... import App
        app_con = App
    app = app_con(name, args=args) if name else app_con()
    logger.info(Style.Bold(f"App instance, returned ID: {app.id}"))

    registered_apps[0] = app
    return app

System Utilities & Configuration

toolboxv2.FileHandler

Bases: Code

Source code in toolboxv2/utils/system/file_handler.py

class FileHandler(Code):

    def __init__(self, filename, name='mainTool', keys=None, defaults=None):
        if defaults is None:
            defaults = {}
        if keys is None:
            keys = {}
        assert filename.endswith(".config") or filename.endswith(".data"), \
            f"filename must end with .config or .data {filename=}"
        self.file_handler_save = {}
        self.file_handler_load = {}
        self.file_handler_key_mapper = {}
        self.file_handler_filename = filename
        self.file_handler_storage = None
        self.file_handler_max_loaded_index_ = 0
        self.file_handler_file_prefix = (f".{filename.split('.')[1]}/"
                                         f"{name.replace('.', '-')}/")
        # self.load_file_handler()
        self.set_defaults_keys_file_handler(keys, defaults)

    def _open_file_handler(self, mode: str, rdu):
        logger = get_logger()
        logger.info(Style.Bold(Style.YELLOW(f"Opening file in mode : {mode}")))
        if self.file_handler_storage:
            self.file_handler_storage.close()
            self.file_handler_storage = None
        try:
            self.file_handler_storage = open(self.file_handler_file_prefix + self.file_handler_filename, mode)
            self.file_handler_max_loaded_index_ += 1
        except FileNotFoundError:
            if self.file_handler_max_loaded_index_ == 2:
                os.makedirs(self.file_handler_file_prefix, exist_ok=True)
            if self.file_handler_max_loaded_index_ == 3:
                os.makedirs(".config/mainTool", exist_ok=True)
            if self.file_handler_max_loaded_index_ >= 5:
                print(Style.RED(f"pleas create this file to prosed : {self.file_handler_file_prefix}"
                                f"{self.file_handler_filename}"))
                logger.critical(f"{self.file_handler_file_prefix} {self.file_handler_filename} FileNotFoundError cannot"
                                f" be Created")
                exit(0)
            self.file_handler_max_loaded_index_ += 1
            logger.info(Style.YELLOW(f"Try Creating File: {self.file_handler_file_prefix}{self.file_handler_filename}"))

            if not os.path.exists(f"{self.file_handler_file_prefix}"):
                os.makedirs(f"{self.file_handler_file_prefix}")

            with open(self.file_handler_file_prefix + self.file_handler_filename, 'a'):
                logger.info(Style.GREEN("File created successfully"))
                self.file_handler_max_loaded_index_ = -1
            rdu()
        except OSError and PermissionError as e:
            raise e

    def open_s_file_handler(self):
        self._open_file_handler('w+', self.open_s_file_handler)
        return self

    def open_l_file_handler(self):
        self._open_file_handler('r+', self.open_l_file_handler)
        return self

    def save_file_handler(self):
        get_logger().info(
            Style.BLUE(
                f"init Saving (S) {self.file_handler_filename} "
            )
        )
        if self.file_handler_storage:
            get_logger().warning(
                f"WARNING file is already open (S): {self.file_handler_filename} {self.file_handler_storage}")

        self.open_s_file_handler()

        get_logger().info(
            Style.BLUE(
                f"Elements to save : ({len(self.file_handler_save.keys())})"
            )
        )

        self.file_handler_storage.write(json.dumps(self.file_handler_save))

        self.file_handler_storage.close()
        self.file_handler_storage = None

        get_logger().info(
            Style.BLUE(
                f"closing file : {self.file_handler_filename} "
            )
        )

        return self

    def add_to_save_file_handler(self, key: str, value: str):
        if len(key) != 10:
            get_logger(). \
                warning(
                Style.YELLOW(
                    'WARNING: key length is not 10 characters'
                )
            )
            return False
        if key not in self.file_handler_load:
            if key in self.file_handler_key_mapper:
                key = self.file_handler_key_mapper[key]

        self.file_handler_load[key] = value
        self.file_handler_save[key] = self.encode_code(value)
        return True

    def remove_key_file_handler(self, key: str):
        if key == 'Pka7237327':
            print("Cant remove Root Key")
            return
        if key in self.file_handler_load:
            del self.file_handler_load[key]
        if key in self.file_handler_save:
            del self.file_handler_save[key]

    def load_file_handler(self):
        get_logger().info(
            Style.BLUE(
                f"loading {self.file_handler_filename} "
            )
        )
        if self.file_handler_storage:
            get_logger().warning(
                Style.YELLOW(
                    f"WARNING file is already open (L) {self.file_handler_filename}"
                )
            )
        self.open_l_file_handler()

        try:

            self.file_handler_save = json.load(self.file_handler_storage)
            for key, line in self.file_handler_save.items():
                self.file_handler_load[key] = self.decode_code(line)

        except json.decoder.JSONDecodeError and Exception:

            for line in self.file_handler_storage:
                line = line[:-1]
                heda = line[:10]
                self.file_handler_save[heda] = line[10:]
                enc = self.decode_code(line[10:])
                self.file_handler_load[heda] = enc

            self.file_handler_save = {}

        self.file_handler_storage.close()
        self.file_handler_storage = None

        return self

    def get_file_handler(self, obj: str, default=None) -> str or None:
        logger = get_logger()
        if obj not in self.file_handler_load:
            if obj in self.file_handler_key_mapper:
                obj = self.file_handler_key_mapper[obj]
        logger.info(Style.ITALIC(Style.GREY(f"Collecting data from storage key : {obj}")))
        self.file_handler_max_loaded_index_ = -1
        for objects in self.file_handler_load.items():
            self.file_handler_max_loaded_index_ += 1
            if obj == objects[0]:

                try:
                    if len(objects[1]) > 0:
                        return ast.literal_eval(objects[1]) if isinstance(objects[1], str) else objects[1]
                    logger.warning(
                        Style.YELLOW(
                            f"No data  {obj}  ; {self.file_handler_filename}"
                        )
                    )
                except ValueError:
                    logger.error(f"ValueError Loading {obj} ; {self.file_handler_filename}")
                except SyntaxError:
                    if isinstance(objects[1], str):
                        return objects[1]
                    logger.warning(
                        Style.YELLOW(
                            f"Possible SyntaxError Loading {obj} ; {self.file_handler_filename}"
                            f" {len(objects[1])} {type(objects[1])}"
                        )
                    )
                    return objects[1]
                except NameError:
                    return str(objects[1])

        if obj in list(self.file_handler_save.keys()):
            r = self.decode_code(self.file_handler_save[obj])
            logger.info(f"returning Default for {obj}")
            return r

        if default is None:
            default = self.file_handler_load.get(obj)

        logger.info("no data found")
        return default

    def set_defaults_keys_file_handler(self, keys: dict, defaults: dict):
        list_keys = iter(list(keys.keys()))
        df_keys = defaults.keys()
        for key in list_keys:
            self.file_handler_key_mapper[key] = keys[key]
            self.file_handler_key_mapper[keys[key]] = key
            if key in df_keys:
                self.file_handler_load[keys[key]] = str(defaults[key])
                self.file_handler_save[keys[key]] = self.encode_code(defaults[key])
            else:
                self.file_handler_load[keys[key]] = "None"

    def delete_file(self):
        os.remove(self.file_handler_file_prefix + self.file_handler_filename)
        get_logger().warning(Style.GREEN(f"File deleted {self.file_handler_file_prefix + self.file_handler_filename}"))

toolboxv2.utils

App

Source code in toolboxv2/utils/toolbox.py

class App(AppType, metaclass=Singleton):

    def __init__(self, prefix: str = "", args=AppArgs().default()):
        if "test" not in prefix:
            prefix = "main"
        super().__init__(prefix, args)
        self._web_context = None
        t0 = time.perf_counter()
        abspath = os.path.abspath(__file__)
        self.system_flag = system()  # Linux: Linux Mac: Darwin Windows: Windows

        self.appdata = os.getenv('APPDATA') if os.name == 'nt' else os.getenv('XDG_CONFIG_HOME') or os.path.expanduser(
                '~/.config') if os.name == 'posix' else None

        if self.system_flag == "Darwin" or self.system_flag == "Linux":
            dir_name = os.path.dirname(abspath).replace("/utils", "")
        else:
            dir_name = os.path.dirname(abspath).replace("\\utils", "")

        self.start_dir = str(dir_name)

        self.bg_tasks = []

        lapp = dir_name + '\\.data\\'

        if not prefix:
            if not os.path.exists(f"{lapp}last-app-prefix.txt"):
                os.makedirs(lapp, exist_ok=True)
                open(f"{lapp}last-app-prefix.txt", "a").close()
            with open(f"{lapp}last-app-prefix.txt") as prefix_file:
                cont = prefix_file.read()
                if cont:
                    prefix = cont.rstrip()
        else:
            if not os.path.exists(f"{lapp}last-app-prefix.txt"):
                os.makedirs(lapp, exist_ok=True)
                open(f"{lapp}last-app-prefix.txt", "a").close()
            with open(f"{lapp}last-app-prefix.txt", "w") as prefix_file:
                prefix_file.write(prefix)

        self.prefix = prefix

        node_ = node()

        if 'localhost' in node_ and (host := os.getenv('HOSTNAME', 'localhost')) != 'localhost':
            node_ = node_.replace('localhost', host)
        self.id = prefix + '-' + node_
        self.globals = {
            "root": {**globals()},
        }
        self.locals = {
            "user": {'app': self, **locals()},
        }

        identification = self.id
        collective_identification = self.id
        if "test" in prefix:
            if self.system_flag == "Darwin" or self.system_flag == "Linux":
                start_dir = self.start_dir.replace("ToolBoxV2/toolboxv2", "toolboxv2")
            else:
                start_dir = self.start_dir.replace("ToolBoxV2\\toolboxv2", "toolboxv2")
            self.data_dir = start_dir + '\\.data\\' + "test"
            self.config_dir = start_dir + '\\.config\\' + "test"
            self.info_dir = start_dir + '\\.info\\' + "test"
        elif identification.startswith('collective-'):
            collective_identification = identification.split('-')[1]
            self.data_dir = self.start_dir + '\\.data\\' + collective_identification
            self.config_dir = self.start_dir + '\\.config\\' + collective_identification
            self.info_dir = self.start_dir + '\\.info\\' + collective_identification
            self.id = collective_identification
        else:
            self.data_dir = self.start_dir + '\\.data\\' + identification
            self.config_dir = self.start_dir + '\\.config\\' + identification
            self.info_dir = self.start_dir + '\\.info\\' + identification

        if self.appdata is None:
            self.appdata = self.data_dir
        else:
            self.appdata += "/ToolBoxV2"

        if not os.path.exists(self.appdata):
            os.makedirs(self.appdata, exist_ok=True)
        if not os.path.exists(self.data_dir):
            os.makedirs(self.data_dir, exist_ok=True)
        if not os.path.exists(self.config_dir):
            os.makedirs(self.config_dir, exist_ok=True)
        if not os.path.exists(self.info_dir):
            os.makedirs(self.info_dir, exist_ok=True)

        self.print(f"Starting ToolBox as {prefix} from :", Style.Bold(Style.CYAN(f"{os.getcwd()}")))

        logger_info_str, self.logger, self.logging_filename = self.set_logger(args.debug)

        self.print("Logger " + logger_info_str)
        self.print("================================")
        self.logger.info("Logger initialized")
        get_logger().info(Style.GREEN("Starting Application instance"))
        if args.init and args.init is not None and self.start_dir not in sys.path:
            sys.path.append(self.start_dir)

        __version__ = get_version_from_pyproject()
        self.version = __version__

        self.keys = {
            "MACRO": "macro~~~~:",
            "MACRO_C": "m_color~~:",
            "HELPER": "helper~~~:",
            "debug": "debug~~~~:",
            "id": "name-spa~:",
            "st-load": "mute~load:",
            "comm-his": "comm-his~:",
            "develop-mode": "dev~mode~:",
            "provider::": "provider::",
        }

        defaults = {
            "MACRO": ['Exit'],
            "MACRO_C": {},
            "HELPER": {},
            "debug": args.debug,
            "id": self.id,
            "st-load": False,
            "comm-his": [[]],
            "develop-mode": False,
        }
        self.config_fh = FileHandler(collective_identification + ".config", keys=self.keys, defaults=defaults)
        self.config_fh.load_file_handler()
        self._debug = args.debug
        self.flows = {}
        self.dev_modi = self.config_fh.get_file_handler(self.keys["develop-mode"])
        if self.config_fh.get_file_handler("provider::") is None:
            self.config_fh.add_to_save_file_handler("provider::", "http://localhost:" + str(
                self.args_sto.port) if os.environ.get("HOSTNAME","localhost") == "localhost" else "https://simplecore.app")
        self.functions = {}
        self.modules = {}

        self.interface_type = ToolBoxInterfaces.native
        self.PREFIX = Style.CYAN(f"~{node()}@>")
        self.alive = True
        self.called_exit = False, time.time()

        self.print(f"Infos:\n  {'Name':<8} -> {node()}\n  {'ID':<8} -> {self.id}\n  {'Version':<8} -> {self.version}\n")

        self.logger.info(
            Style.GREEN(
                f"Finish init up in {time.perf_counter() - t0:.2f}s"
            )
        )

        self.args_sto = args
        self.loop = None

        from .system.session import Session
        self.session: Session = Session(self.get_username())
        if len(sys.argv) > 2 and sys.argv[1] == "db":
            return
        from toolboxv2.utils.clis.db_cli_manager import ClusterManager, get_executable_path
        self.cluster_manager = ClusterManager()
        online_list, server_list = self.cluster_manager.status_all(silent=True)
        if not server_list:
            self.cluster_manager.start_all(get_executable_path(), self.version)
            _, server_list = self.cluster_manager.status_all()
        from .extras.blobs import BlobStorage
        self.root_blob_storage = BlobStorage(servers=server_list, storage_directory=self.data_dir+ '\\blob_cache\\')
        self.mkdocs = add_to_app(self)
        # self._start_event_loop()

    def _start_event_loop(self):
        """Starts the asyncio event loop in a separate thread."""
        if self.loop is None:
            self.loop = asyncio.new_event_loop()
            self.loop_thread = threading.Thread(target=self.loop.run_forever, daemon=True)
            self.loop_thread.start()

    def get_username(self, get_input=False, default="loot") -> str:
        user_name = self.config_fh.get_file_handler("ac_user:::")
        if get_input and user_name is None:
            user_name = input("Input your username: ")
            self.config_fh.add_to_save_file_handler("ac_user:::", user_name)
        if user_name is None:
            user_name = default
            self.config_fh.add_to_save_file_handler("ac_user:::", user_name)
        return user_name

    def set_username(self, username):
        return self.config_fh.add_to_save_file_handler("ac_user:::", username)

    @staticmethod
    def exit_main(*args, **kwargs):
        """proxi attr"""

    @staticmethod
    def hide_console(*args, **kwargs):
        """proxi attr"""

    @staticmethod
    def show_console(*args, **kwargs):
        """proxi attr"""

    @staticmethod
    def disconnect(*args, **kwargs):
        """proxi attr"""

    def set_logger(self, debug=False):
        if "test" in self.prefix and not debug:
            logger, logging_filename = setup_logging(logging.NOTSET, name="toolbox-test", interminal=True,
                                                     file_level=logging.NOTSET, app_name=self.id)
            logger_info_str = "in Test Mode"
        elif "live" in self.prefix and not debug:
            logger, logging_filename = setup_logging(logging.DEBUG, name="toolbox-live", interminal=False,
                                                     file_level=logging.WARNING, app_name=self.id)
            logger_info_str = "in Live Mode"
            # setup_logging(logging.WARNING, name="toolbox-live", is_online=True
            #              , online_level=logging.WARNING).info("Logger initialized")
        elif "debug" in self.prefix or self.prefix.endswith("D"):
            self.prefix = self.prefix.replace("-debug", '').replace("debug", '')
            logger, logging_filename = setup_logging(logging.DEBUG, name="toolbox-debug", interminal=True,
                                                     file_level=logging.WARNING, app_name=self.id)
            logger_info_str = "in debug Mode"
            self.debug = True
        elif debug:
            logger, logging_filename = setup_logging(logging.DEBUG, name=f"toolbox-{self.prefix}-debug",
                                                     interminal=True,
                                                     file_level=logging.DEBUG, app_name=self.id)
            logger_info_str = "in args debug Mode"
        else:
            logger, logging_filename = setup_logging(logging.ERROR, name=f"toolbox-{self.prefix}", app_name=self.id)
            logger_info_str = "in Default"

        return logger_info_str, logger, logging_filename

    @property
    def debug(self):
        return self._debug

    @debug.setter
    def debug(self, value):
        if not isinstance(value, bool):
            self.logger.debug(f"Value must be an boolean. is : {value} type of {type(value)}")
            raise ValueError("Value must be an boolean.")

        # self.logger.info(f"Setting debug {value}")
        self._debug = value

    def debug_rains(self, e):
        if self.debug:
            import traceback
            x = "="*5
            x += " DEBUG "
            x += "="*5
            self.print(x)
            self.print(traceback.format_exc())
            self.print(x)
            raise e
        else:
            self.logger.error(f"Error: {e}")
            import traceback
            x = "="*5
            x += " DEBUG "
            x += "="*5
            self.print(x)
            self.print(traceback.format_exc())
            self.print(x)

    def set_flows(self, r):
        self.flows = r

    async def run_flows(self, name, **kwargs):
        from ..flows import flows_dict as flows_dict_func
        if name not in self.flows:
            self.flows = {**self.flows, **flows_dict_func(s=name, remote=True)}
        if name in self.flows:
            if asyncio.iscoroutinefunction(self.flows[name]):
                return await self.flows[name](get_app(from_="runner"), self.args_sto, **kwargs)
            else:
                return self.flows[name](get_app(from_="runner"), self.args_sto, **kwargs)
        else:
            print("Flow not found, active flows:", len(self.flows.keys()))

    def _coppy_mod(self, content, new_mod_dir, mod_name, file_type='py'):

        mode = 'xb'
        self.logger.info(f" coppy mod {mod_name} to {new_mod_dir} size : {sys.getsizeof(content) / 8388608:.3f} mb")

        if not os.path.exists(new_mod_dir):
            os.makedirs(new_mod_dir)
            with open(f"{new_mod_dir}/__init__.py", "w") as nmd:
                nmd.write(f"__version__ = '{self.version}'")

        if os.path.exists(f"{new_mod_dir}/{mod_name}.{file_type}"):
            mode = False

            with open(f"{new_mod_dir}/{mod_name}.{file_type}", 'rb') as d:
                runtime_mod = d.read()  # Testing version but not efficient

            if len(content) != len(runtime_mod):
                mode = 'wb'

        if mode:
            with open(f"{new_mod_dir}/{mod_name}.{file_type}", mode) as f:
                f.write(content)

    def _pre_lib_mod(self, mod_name, path_to="./runtime", file_type='py'):
        working_dir = self.id.replace(".", "_")
        lib_mod_dir = f"toolboxv2.runtime.{working_dir}.mod_lib."

        self.logger.info(f"pre_lib_mod {mod_name} from {lib_mod_dir}")

        postfix = "_dev" if self.dev_modi else ""
        mod_file_dir = f"./mods{postfix}/{mod_name}.{file_type}"
        new_mod_dir = f"{path_to}/{working_dir}/mod_lib"
        with open(mod_file_dir, "rb") as c:
            content = c.read()
        self._coppy_mod(content, new_mod_dir, mod_name, file_type=file_type)
        return lib_mod_dir

    def _copy_load(self, mod_name, file_type='py', **kwargs):
        loc = self._pre_lib_mod(mod_name, file_type)
        return self.inplace_load_instance(mod_name, loc=loc, **kwargs)

    def helper_install_pip_module(self, module_name):
        if 'main' in self.id:
            return
        self.print(f"Installing {module_name} GREEDY")
        os.system(f"{sys.executable} -m pip install {module_name}")

    def python_module_import_classifier(self, mod_name, error_message):

        if error_message.startswith("No module named 'toolboxv2.utils"):
            return Result.default_internal_error(f"404 {error_message.split('utils')[1]} not found")
        if error_message.startswith("No module named 'toolboxv2.mods"):
            if mod_name.startswith('.'):
                return
            return self.run_a_from_sync(self.a_run_any, ("CloudM", "install"), module_name=mod_name)
        if error_message.startswith("No module named '"):
            pip_requ = error_message.split("'")[1].replace("'", "").strip()
            # if 'y' in input(f"\t\t\tAuto install {pip_requ} Y/n").lower:
            return self.helper_install_pip_module(pip_requ)
            # return Result.default_internal_error(f"404 {pip_requ} not found")

    def inplace_load_instance(self, mod_name, loc="toolboxv2.mods.", spec='app', save=True, mfo=None):
        if self.dev_modi and loc == "toolboxv2.mods.":
            loc = "toolboxv2.mods_dev."
        if spec=='app' and self.mod_online(mod_name):
            self.logger.info(f"Reloading mod from : {loc + mod_name}")
            self.remove_mod(mod_name, spec=spec, delete=False)

        if (os.path.exists(self.start_dir + '/mods/' + mod_name) or os.path.exists(
            self.start_dir + '/mods/' + mod_name + '.py')) and (
            os.path.isdir(self.start_dir + '/mods/' + mod_name) or os.path.isfile(
            self.start_dir + '/mods/' + mod_name + '.py')):
            try:
                if mfo is None:
                    modular_file_object = import_module(loc + mod_name)
                else:
                    modular_file_object = mfo
                self.modules[mod_name] = modular_file_object
            except ModuleNotFoundError as e:
                self.logger.error(Style.RED(f"module {loc + mod_name} not found is type sensitive {e}"))
                self.print(Style.RED(f"module {loc + mod_name} not found is type sensitive {e}"))
                if self.debug or self.args_sto.sysPrint:
                    self.python_module_import_classifier(mod_name, str(e))
                self.debug_rains(e)
                return None
        else:
            self.sprint(f"module {loc + mod_name} is not valid")
            return None
        if hasattr(modular_file_object, "Tools"):
            tools_class = modular_file_object.Tools
        else:
            if hasattr(modular_file_object, "name"):
                tools_class = modular_file_object
                modular_file_object = import_module(loc + mod_name)
            else:
                tools_class = None

        modular_id = None
        instance = modular_file_object
        app_instance_type = "file/application"

        if tools_class is None:
            modular_id = modular_file_object.Name if hasattr(modular_file_object, "Name") else mod_name

        if tools_class is None and modular_id is None:
            modular_id = str(modular_file_object.__name__)
            self.logger.warning(f"Unknown instance loaded {mod_name}")
            return modular_file_object

        if tools_class is not None:
            tools_class = self.save_initialized_module(tools_class, spec)
            modular_id = tools_class.name
            app_instance_type = "functions/class"
        else:
            instance.spec = spec
        # if private:
        #     self.functions[modular_id][f"{spec}_private"] = private

        if not save:
            return instance if tools_class is None else tools_class

        return self.save_instance(instance, modular_id, spec, app_instance_type, tools_class=tools_class)

    def save_instance(self, instance, modular_id, spec='app', instance_type="file/application", tools_class=None):

        if modular_id in self.functions and tools_class is None:
            if self.functions[modular_id].get(f"{spec}_instance", None) is None:
                self.functions[modular_id][f"{spec}_instance"] = instance
                self.functions[modular_id][f"{spec}_instance_type"] = instance_type
            else:
                self.print("Firest instance stays use new spec to get new instance")
                if modular_id in self.functions and self.functions[modular_id].get(f"{spec}_instance", None) is not None:
                    return self.functions[modular_id][f"{spec}_instance"]
                else:
                    raise ImportError(f"Module already known {modular_id} and not avalabel reload using other spec then {spec}")

        elif tools_class is not None:
            if modular_id not in self.functions:
                self.functions[modular_id] = {}
            self.functions[modular_id][f"{spec}_instance"] = tools_class
            self.functions[modular_id][f"{spec}_instance_type"] = instance_type

            try:
                if not hasattr(tools_class, 'tools'):
                    tools_class.tools = {"Version": tools_class.get_version, 'name': tools_class.name}
                for function_name in list(tools_class.tools.keys()):
                    t_function_name = function_name.lower()
                    if t_function_name != "all" and t_function_name != "name":
                        self.tb(function_name, mod_name=modular_id)(tools_class.tools.get(function_name))
                self.functions[modular_id][f"{spec}_instance_type"] += "/BC"
                if hasattr(tools_class, 'on_exit'):
                    if "on_exit" in self.functions[modular_id]:
                        self.functions[modular_id]["on_exit"].append(tools_class.on_exit)
                    else:
                        self.functions[modular_id]["on_exit"] = [tools_class.on_exit]
            except Exception as e:
                self.logger.error(f"Starting Module {modular_id} compatibility failed with : {e}")
                pass
        elif modular_id not in self.functions and tools_class is None:
            self.functions[modular_id] = {}
            self.functions[modular_id][f"{spec}_instance"] = instance
            self.functions[modular_id][f"{spec}_instance_type"] = instance_type

        else:
            raise ImportError(f"Modular {modular_id} is not a valid mod")
        on_start = self.functions[modular_id].get("on_start")
        if on_start is not None:
            i = 1
            for f in on_start:
                try:
                    f_, e = self.get_function((modular_id, f), state=True, specification=spec)
                    if e == 0:
                        self.logger.info(Style.GREY(f"Running On start {f} {i}/{len(on_start)}"))
                        if asyncio.iscoroutinefunction(f_):
                            self.print(f"Async on start is only in Tool claas supported for {modular_id}.{f}" if tools_class is None else f"initialization starting soon for {modular_id}.{f}")
                            self.run_bg_task_advanced(f_)
                        else:
                            o = f_()
                            if o is not None:
                                self.print(f"Function {modular_id} On start result: {o}")
                    else:
                        self.logger.warning(f"starting function not found {e}")
                except Exception as e:
                    self.logger.debug(Style.YELLOW(
                        Style.Bold(f"modular:{modular_id}.{f} on_start error {i}/{len(on_start)} -> {e}")))
                    self.debug_rains(e)
                finally:
                    i += 1
        return instance if tools_class is None else tools_class

    def save_initialized_module(self, tools_class, spec):
        tools_class.spec = spec
        live_tools_class = tools_class(app=self)
        return live_tools_class

    def mod_online(self, mod_name, installed=False):
        if installed and mod_name not in self.functions:
            self.save_load(mod_name)
        return mod_name in self.functions

    def _get_function(self,
                      name: Enum or None,
                      state: bool = True,
                      specification: str = "app",
                      metadata=False, as_str: tuple or None = None, r=0, **kwargs):

        if as_str is None and isinstance(name, Enum):
            modular_id = str(name.NAME.value)
            function_id = str(name.value)
        elif as_str is None and isinstance(name, list):
            modular_id, function_id = name[0], name[1]
        else:
            modular_id, function_id = as_str

        self.logger.info(f"getting function : {specification}.{modular_id}.{function_id}")

        if modular_id not in self.functions:
            if r == 0:
                self.save_load(modular_id, spec=specification)
                return self.get_function(name=(modular_id, function_id),
                                         state=state,
                                         specification=specification,
                                         metadata=metadata,
                                         r=1)
            self.logger.warning(f"function modular not found {modular_id} 404")
            return "404", 404

        if function_id not in self.functions[modular_id]:
            self.logger.warning(f"function data not found {modular_id}.{function_id} 404")
            return "404", 404

        function_data = self.functions[modular_id][function_id]

        if isinstance(function_data, list):
            print(f"functions {function_id} : {function_data}")
            function_data = self.functions[modular_id][function_data[kwargs.get('i', -1)]]
            print(f"functions {modular_id} : {function_data}")
        function = function_data.get("func")
        params = function_data.get("params")

        state_ = function_data.get("state")
        if state_ is not None and state != state_:
            state = state_

        if function is None:
            self.logger.warning("No function found")
            return "404", 404

        if params is None:
            self.logger.warning("No function (params) found")
            return "404", 301

        if metadata and not state:
            self.logger.info("returning metadata stateless")
            return (function_data, function), 0

        if not state:  # mens a stateless function
            self.logger.info("returning stateless function")
            return function, 0

        instance = self.functions[modular_id].get(f"{specification}_instance")

        # instance_type = self.functions[modular_id].get(f"{specification}_instance_type", "functions/class")

        if params[0] == 'app':
            instance = get_app(from_=f"fuction {specification}.{modular_id}.{function_id}")

        if instance is None and self.alive:
            self.inplace_load_instance(modular_id, spec=specification)
            instance = self.functions[modular_id].get(f"{specification}_instance")

        if instance is None:
            self.logger.warning("No live Instance found")
            return "404", 400

        # if instance_type.endswith("/BC"):  # for backwards compatibility  functions/class/BC old modules
        #     # returning as stateless
        #     # return "422", -1
        #     self.logger.info(
        #         f"returning stateless function, cant find tools class for state handling found {instance_type}")
        #     if metadata:
        #         self.logger.info(f"returning metadata stateless")
        #         return (function_data, function), 0
        #     return function, 0

        self.logger.info("wrapping in higher_order_function")

        self.logger.info(f"returned fuction {specification}.{modular_id}.{function_id}")
        higher_order_function = partial(function, instance)

        if metadata:
            self.logger.info("returning metadata stateful")
            return (function_data, higher_order_function), 0

        self.logger.info("returning stateful function")
        return higher_order_function, 0

    def save_exit(self):
        self.logger.info(f"save exiting saving data to {self.config_fh.file_handler_filename} states of {self.debug=}")
        self.config_fh.add_to_save_file_handler(self.keys["debug"], str(self.debug))

    def init_mod(self, mod_name, spec='app'):
        """
        Initializes a module in a thread-safe manner by submitting the
        asynchronous initialization to the running event loop.
        """
        if '.' in mod_name:
            mod_name = mod_name.split('.')[0]
        self.run_bg_task(self.a_init_mod, mod_name, spec)
        # loop = self.loop_gard()
        # if loop:
        #     # Create a future to get the result from the coroutine
        #     future: Future = asyncio.run_coroutine_threadsafe(
        #         self.a_init_mod(mod_name, spec), loop
        #     )
        #     # Block until the result is available
        #     return future.result()
        # else:
        #     raise ValueError("Event loop is not running")
        #     # return self.loop_gard().run_until_complete(self.a_init_mod(mod_name, spec))

    def run_bg_task(self, task: Callable, *args, **kwargs) -> asyncio.Task | None:
        """
        Runs a coroutine in the background without blocking the caller.

        This is the primary method for "fire-and-forget" async tasks. It schedules
        the coroutine to run on the application's main event loop.

        Args:
            task: The coroutine function to run.
            *args: Arguments to pass to the coroutine function.
            **kwargs: Keyword arguments to pass to the coroutine function.

        Returns:
            An asyncio.Task object representing the scheduled task, or None if
            the task could not be scheduled.
        """
        if not callable(task):
            self.logger.warning("Task passed to run_bg_task is not callable!")
            return None

        if not asyncio.iscoroutinefunction(task) and not asyncio.iscoroutine(task):
            self.logger.warning(f"Task '{getattr(task, '__name__', 'unknown')}' is not a coroutine. "
                                f"Use run_bg_task_advanced for synchronous functions.")
            # Fallback to advanced runner for convenience
            self.run_bg_task_advanced(task, *args, **kwargs)
            return None

        try:
            loop = self.loop_gard()
            if not loop.is_running():
                # If the main loop isn't running, we can't create a task on it.
                # This scenario is handled by run_bg_task_advanced.
                self.logger.info("Main event loop not running. Delegating to advanced background runner.")
                return self.run_bg_task_advanced(task, *args, **kwargs)

            # Create the coroutine if it's a function
            coro = task(*args, **kwargs) if asyncio.iscoroutinefunction(task) else task

            # Create a task on the running event loop
            bg_task = loop.create_task(coro)

            # Add a callback to log exceptions from the background task
            def _log_exception(the_task: asyncio.Task):
                if not the_task.cancelled() and the_task.exception():
                    self.logger.error(f"Exception in background task '{the_task.get_name()}':",
                                      exc_info=the_task.exception())

            bg_task.add_done_callback(_log_exception)
            self.bg_tasks.append(bg_task)
            return bg_task

        except Exception as e:
            self.logger.error(f"Failed to schedule background task: {e}", exc_info=True)
            return None

    def run_bg_task_advanced(self, task: Callable, *args, **kwargs) -> threading.Thread:
        """
        Runs a task in a separate, dedicated background thread with its own event loop.

        This is ideal for:
        1. Running an async task from a synchronous context.
        2. Launching a long-running, independent operation that should not
           interfere with the main application's event loop.

        Args:
            task: The function to run (can be sync or async).
            *args: Arguments for the task.
            **kwargs: Keyword arguments for the task.

        Returns:
            The threading.Thread object managing the background execution.
        """
        if not callable(task):
            self.logger.warning("Task for run_bg_task_advanced is not callable!")
            return None

        def thread_target():
            # Each thread gets its own event loop.
            loop = asyncio.new_event_loop()
            asyncio.set_event_loop(loop)

            try:
                # Prepare the coroutine we need to run
                if asyncio.iscoroutinefunction(task):
                    coro = task(*args, **kwargs)
                elif asyncio.iscoroutine(task):
                    # It's already a coroutine object
                    coro = task
                else:
                    # It's a synchronous function, run it in an executor
                    # to avoid blocking the new event loop.
                    coro = loop.run_in_executor(None, lambda: task(*args, **kwargs))

                # Run the coroutine to completion
                result = loop.run_until_complete(coro)
                self.logger.debug(f"Advanced background task '{getattr(task, '__name__', 'unknown')}' completed.")
                if result is not None:
                    self.logger.debug(f"Task result: {str(result)[:100]}")

            except Exception as e:
                self.logger.error(f"Error in advanced background task '{getattr(task, '__name__', 'unknown')}':",
                                  exc_info=e)
            finally:
                # Cleanly shut down the event loop in this thread.
                try:
                    all_tasks = asyncio.all_tasks(loop=loop)
                    if all_tasks:
                        for t in all_tasks:
                            t.cancel()
                        loop.run_until_complete(asyncio.gather(*all_tasks, return_exceptions=True))
                finally:
                    loop.close()
                    asyncio.set_event_loop(None)

        # Create, start, and return the thread.
        # It's a daemon thread so it won't prevent the main app from exiting.
        t = threading.Thread(target=thread_target, daemon=True, name=f"BGTask-{getattr(task, '__name__', 'unknown')}")
        self.bg_tasks.append(t)
        t.start()
        return t

    # Helper method to wait for background tasks to complete (optional)
    def wait_for_bg_tasks(self, timeout=None):
        """
        Wait for all background tasks to complete.

        Args:
            timeout: Maximum time to wait (in seconds) for all tasks to complete.
                     None means wait indefinitely.

        Returns:
            bool: True if all tasks completed, False if timeout occurred
        """
        active_tasks = [t for t in self.bg_tasks if t.is_alive()]

        for task in active_tasks:
            task.join(timeout=timeout)
            if task.is_alive():
                return False

        return True

    def __call__(self, *args, **kwargs):
        return self.run(*args, **kwargs)

    def run(self, *args, request=None, running_function_coro=None, **kwargs):
        """
        Run a function with support for SSE streaming in both
        threaded and non-threaded contexts.
        """
        if running_function_coro is None:
            mn, fn = args[0]
            if self.functions.get(mn, {}).get(fn, {}).get('request_as_kwarg', False):
                kwargs["request"] = RequestData.from_dict(request)
                if 'data' in kwargs and 'data' not in self.functions.get(mn, {}).get(fn, {}).get('params', []):
                    kwargs["request"].data = kwargs["request"].body = kwargs['data']
                    del kwargs['data']
                if 'form_data' in kwargs and 'form_data' not in self.functions.get(mn, {}).get(fn, {}).get('params',
                                                                                                           []):
                    kwargs["request"].form_data = kwargs["request"].body = kwargs['form_data']
                    del kwargs['form_data']
            else:
                params = self.functions.get(mn, {}).get(fn, {}).get('params', [])
                # auto pars data and form_data to kwargs by key
                do = False
                data = {}
                if 'data' in kwargs and 'data' not in params:
                    do = True
                    data = kwargs['data']
                    del kwargs['data']
                if 'form_data' in kwargs and 'form_data' not in params:
                    do = True
                    data = kwargs['form_data']
                    del kwargs['form_data']
                if do:
                    for k in params:
                        if k in data:
                            kwargs[k] = data[k]
                            del data[k]

        # Create the coroutine
        coro = running_function_coro or self.a_run_any(*args, **kwargs)

        # Get or create an event loop
        try:
            loop = asyncio.get_event_loop()
            is_running = loop.is_running()
        except RuntimeError:
            loop = asyncio.new_event_loop()
            asyncio.set_event_loop(loop)
            is_running = False

        # If the loop is already running, run in a separate thread
        if is_running:
            # Create thread pool executor as needed
            if not hasattr(self.__class__, '_executor'):
                self.__class__._executor = ThreadPoolExecutor(max_workers=4)

            def run_in_new_thread():
                # Set up a new loop in this thread
                new_loop = asyncio.new_event_loop()
                asyncio.set_event_loop(new_loop)

                try:
                    # Run the coroutine
                    return new_loop.run_until_complete(coro)
                finally:
                    new_loop.close()

            # Run in thread and get result
            thread_result = self.__class__._executor.submit(run_in_new_thread).result()

            # Handle streaming results from thread
            if isinstance(thread_result, dict) and thread_result.get("is_stream"):
                # Create a new SSE stream in the main thread
                async def stream_from_function():
                    # Re-run the function with direct async access
                    stream_result = await self.a_run_any(*args, **kwargs)

                    if (isinstance(stream_result, Result) and
                        getattr(stream_result.result, 'data_type', None) == "stream"):
                        # Get and forward data from the original generator
                        original_gen = stream_result.result.data.get("generator")
                        if inspect.isasyncgen(original_gen):
                            async for item in original_gen:
                                yield item

                # Return a new streaming Result
                return Result.stream(
                    stream_generator=stream_from_function(),
                    headers=thread_result.get("headers", {})
                )

            result = thread_result
        else:
            # Direct execution when loop is not running
            result = loop.run_until_complete(coro)

        # Process the final result
        if isinstance(result, Result):
            if 'debug' in self.id:
                result.print()
            if getattr(result.result, 'data_type', None) == "stream":
                return result
            return result.to_api_result().model_dump(mode='json')

        return result

    def loop_gard(self):
        if self.loop is None:
            self._start_event_loop()
            self.loop = asyncio.get_event_loop()
        if self.loop.is_closed():
            self.loop = asyncio.get_event_loop()
        return self.loop

    async def a_init_mod(self, mod_name, spec='app'):
        mod = self.save_load(mod_name, spec=spec)
        if hasattr(mod, "__initobj") and not mod.async_initialized:
            await mod
        return mod


    def load_mod(self, mod_name: str, mlm='I', **kwargs):

        action_list_helper = ['I (inplace load dill on error python)',
                              # 'C (coppy py file to runtime dir)',
                              # 'S (save py file to dill)',
                              # 'CS (coppy and save py file)',
                              # 'D (development mode, inplace load py file)'
                              ]
        action_list = {"I": lambda: self.inplace_load_instance(mod_name, **kwargs),
                       "C": lambda: self._copy_load(mod_name, **kwargs)
                       }

        try:
            if mlm in action_list:

                return action_list.get(mlm)()
            else:
                self.logger.critical(
                    f"config mlm must be {' or '.join(action_list_helper)} is {mlm=}")
                raise ValueError(f"config mlm must be {' or '.join(action_list_helper)} is {mlm=}")
        except ValueError as e:
            self.logger.warning(Style.YELLOW(f"Error Loading Module '{mod_name}', with error :{e}"))
            self.debug_rains(e)
        except ImportError as e:
            self.logger.error(Style.YELLOW(f"Error Loading Module '{mod_name}', with error :{e}"))
            self.debug_rains(e)
        except Exception as e:
            self.logger.critical(Style.RED(f"Error Loading Module '{mod_name}', with critical error :{e}"))
            print(Style.RED(f"Error Loading Module '{mod_name}'"))
            self.debug_rains(e)

        return Result.default_internal_error(info="info's in logs.")

    async def load_external_mods(self):
        for mod_path in os.getenv("EXTERNAL_PATH_RUNNABLE", '').split(','):
            if mod_path:
                await self.load_all_mods_in_file(mod_path)

    async def load_all_mods_in_file(self, working_dir="mods"):
        print(f"LOADING ALL MODS FROM FOLDER : {working_dir}")
        t0 = time.perf_counter()
        # Get the list of all modules
        module_list = self.get_all_mods(working_dir)
        open_modules = self.functions.keys()
        start_len = len(open_modules)

        for om in open_modules:
            if om in module_list:
                module_list.remove(om)

        tasks: set[Task] = set()

        _ = {tasks.add(asyncio.create_task(asyncio.to_thread(self.save_load, mod, 'app'))) for mod in module_list}
        for t in asyncio.as_completed(tasks):
            try:
                result = await t
                if hasattr(result, 'Name'):
                    self.print('Opened :', result.Name)
                elif hasattr(result, 'name'):
                    if hasattr(result, 'async_initialized'):
                        if not result.async_initialized:
                            async def _():
                                try:
                                    if asyncio.iscoroutine(result):
                                        await result
                                    if hasattr(result, 'Name'):
                                        self.print('Opened :', result.Name)
                                    elif hasattr(result, 'name'):
                                        self.print('Opened :', result.name)
                                except Exception as e:
                                    self.debug_rains(e)
                                    if hasattr(result, 'Name'):
                                        self.print('Error opening :', result.Name)
                                    elif hasattr(result, 'name'):
                                        self.print('Error opening :', result.name)
                            asyncio.create_task(_())
                        else:
                            self.print('Opened :', result.name)
                else:
                    if result:
                        self.print('Opened :', result)
            except Exception as e:
                self.logger.error(Style.RED(f"An Error occurred while opening all modules error: {str(e)}"))
                self.debug_rains(e)
        opened = len(self.functions.keys()) - start_len

        self.logger.info(f"Opened {opened} modules in {time.perf_counter() - t0:.2f}s")
        return f"Opened {opened} modules in {time.perf_counter() - t0:.2f}s"

    def get_all_mods(self, working_dir="mods", path_to="./runtime", use_wd=True):
        self.logger.info(f"collating all mods in working directory {working_dir}")

        pr = "_dev" if self.dev_modi else ""
        if working_dir == "mods" and use_wd:
            working_dir = f"{self.start_dir}/mods{pr}"
        elif use_wd:
            pass
        else:
            w_dir = self.id.replace(".", "_")
            working_dir = f"{path_to}/{w_dir}/mod_lib{pr}/"
        res = os.listdir(working_dir)

        self.logger.info(f"found : {len(res)} files")

        def do_helper(_mod):
            if "mainTool" in _mod:
                return False
            # if not _mod.endswith(".py"):
            #     return False
            if _mod.startswith("__"):
                return False
            if _mod.startswith("."):
                return False
            return not _mod.startswith("test_")

        def r_endings(word: str):
            if word.endswith(".py"):
                return word[:-3]
            return word

        mods_list = list(map(r_endings, filter(do_helper, res)))

        self.logger.info(f"found : {len(mods_list)} Modules")
        return mods_list

    def remove_all_modules(self, delete=False):
        for mod in list(self.functions.keys()):
            self.logger.info(f"closing: {mod}")
            self.remove_mod(mod, delete=delete)

    def remove_mod(self, mod_name, spec='app', delete=True):
        if mod_name not in self.functions:
            self.logger.info(f"mod not active {mod_name}")
            return

        on_exit = self.functions[mod_name].get("on_exit")
        self.logger.info(f"closing: {on_exit}")
        def helper():
            if f"{spec}_instance" in self.functions[mod_name]:
                del self.functions[mod_name][f"{spec}_instance"]
            if f"{spec}_instance_type" in self.functions[mod_name]:
                del self.functions[mod_name][f"{spec}_instance_type"]

        if on_exit is None and self.functions[mod_name].get(f"{spec}_instance_type", "").endswith("/BC"):
            instance = self.functions[mod_name].get(f"{spec}_instance", None)
            if instance is not None and hasattr(instance, 'on_exit'):
                if asyncio.iscoroutinefunction(instance.on_exit):
                    self.exit_tasks.append(instance.on_exit)
                else:
                    instance.on_exit()

        if on_exit is None and delete:
            self.functions[mod_name] = {}
            del self.functions[mod_name]
            return
        if on_exit is None:
            helper()
            return

        i = 1

        for j, f in enumerate(on_exit):
            try:
                f_, e = self.get_function((mod_name, f), state=True, specification=spec, i=j)
                if e == 0:
                    self.logger.info(Style.GREY(f"Running On exit {f} {i}/{len(on_exit)}"))
                    if asyncio.iscoroutinefunction(f_):
                        self.exit_tasks.append(f_)
                        o = None
                    else:
                        o = f_()
                    if o is not None:
                        self.print(f"Function On Exit result: {o}")
                else:
                    self.logger.warning("closing function not found")
            except Exception as e:
                self.logger.debug(
                    Style.YELLOW(Style.Bold(f"modular:{mod_name}.{f} on_exit error {i}/{len(on_exit)} -> {e}")))

                self.debug_rains(e)
            finally:
                i += 1

        helper()

        if delete:
            self.functions[mod_name] = {}
            del self.functions[mod_name]

    async def a_remove_all_modules(self, delete=False):
        for mod in list(self.functions.keys()):
            self.logger.info(f"closing: {mod}")
            await self.a_remove_mod(mod, delete=delete)

    async def a_remove_mod(self, mod_name, spec='app', delete=True):
        if mod_name not in self.functions:
            self.logger.info(f"mod not active {mod_name}")
            return
        on_exit = self.functions[mod_name].get("on_exit")
        self.logger.info(f"closing: {on_exit}")
        def helper():
            if f"{spec}_instance" in self.functions[mod_name]:
                del self.functions[mod_name][f"{spec}_instance"]
            if f"{spec}_instance_type" in self.functions[mod_name]:
                del self.functions[mod_name][f"{spec}_instance_type"]

        if on_exit is None and self.functions[mod_name].get(f"{spec}_instance_type", "").endswith("/BC"):
            instance = self.functions[mod_name].get(f"{spec}_instance", None)
            if instance is not None and hasattr(instance, 'on_exit'):
                if asyncio.iscoroutinefunction(instance.on_exit):
                    await instance.on_exit()
                else:
                    instance.on_exit()

        if on_exit is None and delete:
            self.functions[mod_name] = {}
            del self.functions[mod_name]
            return
        if on_exit is None:
            helper()
            return

        i = 1
        for f in on_exit:
            try:
                e = 1
                if isinstance(f, str):
                    f_, e = self.get_function((mod_name, f), state=True, specification=spec)
                elif isinstance(f, Callable):
                    f_, e, f  = f, 0, f.__name__
                if e == 0:
                    self.logger.info(Style.GREY(f"Running On exit {f} {i}/{len(on_exit)}"))
                    if asyncio.iscoroutinefunction(f_):
                        o = await f_()
                    else:
                        o = f_()
                    if o is not None:
                        self.print(f"Function On Exit result: {o}")
                else:
                    self.logger.warning("closing function not found")
            except Exception as e:
                self.logger.debug(
                    Style.YELLOW(Style.Bold(f"modular:{mod_name}.{f} on_exit error {i}/{len(on_exit)} -> {e}")))
                self.debug_rains(e)
            finally:
                i += 1

        helper()

        if delete:
            self.functions[mod_name] = {}
            del self.functions[mod_name]

    def exit(self, remove_all=True):
        if not self.alive:
            return
        if self.args_sto.debug:
            self.hide_console()
        self.disconnect()
        if remove_all:
            self.remove_all_modules()
        self.logger.info("Exiting ToolBox interface")
        self.alive = False
        self.called_exit = True, time.time()
        self.save_exit()
        if hasattr(self, 'root_blob_storage') and self.root_blob_storage:
            self.root_blob_storage.exit()
        try:
            self.config_fh.save_file_handler()
        except SystemExit:
            print("If u ar testing this is fine else ...")

        if hasattr(self, 'daemon_app'):
            import threading

            for thread in threading.enumerate()[::-1]:
                if thread.name == "MainThread":
                    continue
                try:
                    with Spinner(f"closing Thread {thread.name:^50}|", symbols="s", count_down=True,
                                 time_in_s=0.751 if not self.debug else 0.6):
                        thread.join(timeout=0.751 if not self.debug else 0.6)
                except TimeoutError as e:
                    self.logger.error(f"Timeout error on exit {thread.name} {str(e)}")
                    print(str(e), f"Timeout {thread.name}")
                except KeyboardInterrupt:
                    print("Unsave Exit")
                    break
        if hasattr(self, 'loop') and self.loop is not None:
            with Spinner("closing Event loop:", symbols="+"):
                self.loop.stop()

    async def a_exit(self):

        import inspect
        self.sprint(f"exit requested from: {inspect.stack()[1].filename}::{inspect.stack()[1].lineno} function: {inspect.stack()[1].function}")

        # Cleanup session before removing modules
        try:
            if hasattr(self, 'session') and self.session is not None:
                await self.session.cleanup()
        except Exception as e:
            self.logger.debug(f"Session cleanup error (ignored): {e}")

        await self.a_remove_all_modules(delete=True)
        results = await asyncio.gather(
            *[asyncio.create_task(f()) for f in self.exit_tasks if asyncio.iscoroutinefunction(f)])
        for result in results:
            self.print(f"Function On Exit result: {result}")
        self.exit(remove_all=False)

    def save_load(self, modname, spec='app'):
        self.logger.debug(f"Save load module {modname}")
        if not modname:
            self.logger.warning("no filename specified")
            return False
        try:
            return self.load_mod(modname, spec=spec)
        except ModuleNotFoundError as e:
            self.logger.error(Style.RED(f"Module {modname} not found"))
            self.debug_rains(e)

        return False

    def get_function(self, name: Enum or tuple, **kwargs):
        """
        Kwargs for _get_function
            metadata:: return the registered function dictionary
                stateless: (function_data, None), 0
                stateful: (function_data, higher_order_function), 0
            state::boolean
                specification::str default app
        """
        if isinstance(name, tuple):
            return self._get_function(None, as_str=name, **kwargs)
        else:
            return self._get_function(name, **kwargs)

    async def a_run_function(self, mod_function_name: Enum or tuple,
                             tb_run_function_with_state=True,
                             tb_run_with_specification='app',
                             args_=None,
                             kwargs_=None,
                             *args,
                             **kwargs) -> Result:

        if kwargs_ is not None and not kwargs:
            kwargs = kwargs_
        if args_ is not None and not args:
            args = args_
        if isinstance(mod_function_name, tuple):
            modular_name, function_name = mod_function_name
        elif isinstance(mod_function_name, list):
            modular_name, function_name = mod_function_name[0], mod_function_name[1]
        elif isinstance(mod_function_name, Enum):
            modular_name, function_name = mod_function_name.__class__.NAME.value, mod_function_name.value
        else:
            raise TypeError("Unknown function type")

        if tb_run_with_specification == 'ws_internal':
            modular_name = modular_name.split('/')[0]
            if not self.mod_online(modular_name, installed=True):
                self.get_mod(modular_name)
            handler_id, event_name = mod_function_name
            if handler_id in self.websocket_handlers and event_name in self.websocket_handlers[handler_id]:
                handler_func = self.websocket_handlers[handler_id][event_name]
                try:
                    # Führe den asynchronen Handler aus
                    if inspect.iscoroutinefunction(handler_func):
                        await handler_func(self, **kwargs)
                    else:
                        handler_func(self, **kwargs)  # Für synchrone Handler
                    return Result.ok(info=f"WS handler '{event_name}' executed.")
                except Exception as e:
                    self.logger.error(f"Error in WebSocket handler '{handler_id}/{event_name}': {e}", exc_info=True)
                    return Result.default_internal_error(info=str(e))
            else:
                # Kein Handler registriert, aber das ist kein Fehler (z.B. on_connect ist optional)
                return Result.ok(info=f"No WS handler for '{event_name}'.")

        if not self.mod_online(modular_name, installed=True):
            self.get_mod(modular_name)

        function_data, error_code = self.get_function(mod_function_name, state=tb_run_function_with_state,
                                                      metadata=True, specification=tb_run_with_specification)
        self.logger.info(f"Received fuction : {mod_function_name}, with execode: {error_code}")
        if error_code == 404:
            mod = self.get_mod(modular_name)
            if hasattr(mod, "async_initialized") and not mod.async_initialized:
                await mod
            function_data, error_code = self.get_function(mod_function_name, state=tb_run_function_with_state,
                                                          metadata=True, specification=tb_run_with_specification)

        if error_code == 404:
            self.logger.warning(Style.RED("Function Not Found"))
            return (Result.default_user_error(interface=self.interface_type,
                                              exec_code=404,
                                              info="function not found function is not decorated").
                    set_origin(mod_function_name))

        if error_code == 300:
            return Result.default_internal_error(interface=self.interface_type,
                                                 info=f"module {modular_name}"
                                                      f" has no state (instance)").set_origin(mod_function_name)

        if error_code != 0:
            return Result.default_internal_error(interface=self.interface_type,
                                                 exec_code=error_code,
                                                 info=f"Internal error"
                                                      f" {modular_name}."
                                                      f"{function_name}").set_origin(mod_function_name)

        if not tb_run_function_with_state:
            function_data, _ = function_data
            function = function_data.get('func')
        else:
            function_data, function = function_data

        if not function:
            self.logger.warning(Style.RED(f"Function {function_name} not found"))
            return Result.default_internal_error(interface=self.interface_type,
                                                 exec_code=404,
                                                 info="function not found function").set_origin(mod_function_name)

        self.logger.info("Profiling function")
        t0 = time.perf_counter()
        if asyncio.iscoroutinefunction(function):
            return await self.a_fuction_runner(function, function_data, args, kwargs, t0)
        else:
            return self.fuction_runner(function, function_data, args, kwargs, t0)


    def run_function(self, mod_function_name: Enum or tuple,
                     tb_run_function_with_state=True,
                     tb_run_with_specification='app',
                     args_=None,
                     kwargs_=None,
                     *args,
                     **kwargs) -> Result:

        if kwargs_ is not None and not kwargs:
            kwargs = kwargs_
        if args_ is not None and not args:
            args = args_
        if isinstance(mod_function_name, tuple):
            modular_name, function_name = mod_function_name
        elif isinstance(mod_function_name, list):
            modular_name, function_name = mod_function_name[0], mod_function_name[1]
        elif isinstance(mod_function_name, Enum):
            modular_name, function_name = mod_function_name.__class__.NAME.value, mod_function_name.value
        else:
            raise TypeError("Unknown function type")

        if not self.mod_online(modular_name, installed=True):
            self.get_mod(modular_name)

        if tb_run_with_specification == 'ws_internal':
            handler_id, event_name = mod_function_name
            if handler_id in self.websocket_handlers and event_name in self.websocket_handlers[handler_id]:
                handler_func = self.websocket_handlers[handler_id][event_name]
                try:
                    # Führe den asynchronen Handler aus
                    if inspect.iscoroutinefunction(handler_func):
                        return self.loop.run_until_complete(handler_func(self, **kwargs))
                    else:
                        handler_func(self, **kwargs)  # Für synchrone Handler
                    return Result.ok(info=f"WS handler '{event_name}' executed.")
                except Exception as e:
                    self.logger.error(f"Error in WebSocket handler '{handler_id}/{event_name}': {e}", exc_info=True)
                    return Result.default_internal_error(info=str(e))
            else:
                # Kein Handler registriert, aber das ist kein Fehler (z.B. on_connect ist optional)
                return Result.ok(info=f"No WS handler for '{event_name}'.")

        function_data, error_code = self.get_function(mod_function_name, state=tb_run_function_with_state,
                                                      metadata=True, specification=tb_run_with_specification)
        self.logger.info(f"Received fuction : {mod_function_name}, with execode: {error_code}")
        if error_code == 1 or error_code == 3 or error_code == 400:
            self.get_mod(modular_name)
            function_data, error_code = self.get_function(mod_function_name, state=tb_run_function_with_state,
                                                          metadata=True, specification=tb_run_with_specification)

        if error_code == 2:
            self.logger.warning(Style.RED("Function Not Found"))
            return (Result.default_user_error(interface=self.interface_type,
                                              exec_code=404,
                                              info="function not found function is not decorated").
                    set_origin(mod_function_name))

        if error_code == -1:
            return Result.default_internal_error(interface=self.interface_type,
                                                 info=f"module {modular_name}"
                                                      f" has no state (instance)").set_origin(mod_function_name)

        if error_code != 0:
            return Result.default_internal_error(interface=self.interface_type,
                                                 exec_code=error_code,
                                                 info=f"Internal error"
                                                      f" {modular_name}."
                                                      f"{function_name}").set_origin(mod_function_name)

        if not tb_run_function_with_state:
            function_data, _ = function_data
            function = function_data.get('func')
        else:
            function_data, function = function_data

        if not function:
            self.logger.warning(Style.RED(f"Function {function_name} not found"))
            return Result.default_internal_error(interface=self.interface_type,
                                                 exec_code=404,
                                                 info="function not found function").set_origin(mod_function_name)

        self.logger.info("Profiling function")
        t0 = time.perf_counter()
        if asyncio.iscoroutinefunction(function):
            try:
                return asyncio.run(self.a_fuction_runner(function, function_data, args, kwargs, t0))
            except RuntimeError:
                try:
                    return self.loop.run_until_complete(self.a_fuction_runner(function, function_data, args, kwargs, t0))
                except RuntimeError:
                    pass
            raise ValueError(f"Fuction {function_name} is Async use a_run_any")
        else:
            return self.fuction_runner(function, function_data, args, kwargs, t0)

    def run_a_from_sync(self, function, *args, **kwargs):
        # Initialize self.loop if not already set.
        if self.loop is None:
            try:
                self.loop = asyncio.get_running_loop()
            except RuntimeError:
                self.loop = asyncio.new_event_loop()

        # If the loop is running, offload the coroutine to a new thread.
        if self.loop.is_running():
            result_future = Future()

            def run_in_new_loop():
                new_loop = asyncio.new_event_loop()
                asyncio.set_event_loop(new_loop)
                try:
                    result = new_loop.run_until_complete(function(*args, **kwargs))
                    result_future.set_result(result)
                except Exception as e:
                    result_future.set_exception(e)
                finally:
                    new_loop.close()

            thread = threading.Thread(target=run_in_new_loop)
            thread.start()
            thread.join()  # Block until the thread completes.
            return result_future.result()
        else:
            # If the loop is not running, schedule and run the coroutine directly.
            future = self.loop.create_task(function(*args, **kwargs))
            return self.loop.run_until_complete(future)

    def fuction_runner(self, function, function_data: dict, args: list, kwargs: dict, t0=.0):

        parameters = function_data.get('params')
        modular_name = function_data.get('module_name')
        function_name = function_data.get('func_name')
        row = function_data.get('row')
        mod_function_name = f"{modular_name}.{function_name}"

        if_self_state = 1 if 'self' in parameters else 0

        try:
            if len(parameters) == 0:
                res = function()
            elif len(parameters) == len(args) + if_self_state:
                res = function(*args)
            elif len(parameters) == len(kwargs.keys()) + if_self_state:
                res = function(**kwargs)
            else:
                res = function(*args, **kwargs)
            self.logger.info(f"Execution done in {time.perf_counter()-t0:.4f}")
            if isinstance(res, Result):
                formatted_result = res
                if formatted_result.origin is None:
                    formatted_result.set_origin(mod_function_name)
            elif isinstance(res, ApiResult):
                formatted_result = res
                if formatted_result.origin is None:
                    formatted_result.as_result().set_origin(mod_function_name).to_api_result()
            elif row:
                formatted_result = res
            else:
                # Wrap the result in a Result object
                formatted_result = Result.ok(
                    interface=self.interface_type,
                    data_info="Auto generated result",
                    data=res,
                    info="Function executed successfully"
                ).set_origin(mod_function_name)
            if not row:
                self.logger.info(
                    f"Function Exec code: {formatted_result.info.exec_code} Info's: {formatted_result.info.help_text}")
            else:
                self.logger.info(
                    f"Function Exec data: {formatted_result}")
        except Exception as e:
            self.logger.error(
                Style.YELLOW(Style.Bold(
                    f"! Function ERROR: in {modular_name}.{function_name}")))
            # Wrap the exception in a Result object
            formatted_result = Result.default_internal_error(info=str(e)).set_origin(mod_function_name)
            # res = formatted_result
            self.logger.error(
                f"Function {modular_name}.{function_name}"
                f" executed wit an error {str(e)}, {type(e)}")
            self.debug_rains(e)
            self.print(f"! Function ERROR: in {modular_name}.{function_name} ")



        else:
            self.print_ok()

            self.logger.info(
                f"Function {modular_name}.{function_name}"
                f" executed successfully")

        return formatted_result

    async def a_fuction_runner(self, function, function_data: dict, args: list, kwargs: dict, t0=.0):

        parameters = function_data.get('params')
        modular_name = function_data.get('module_name')
        function_name = function_data.get('func_name')
        row = function_data.get('row')
        mod_function_name = f"{modular_name}.{function_name}"

        if_self_state = 1 if 'self' in parameters else 0

        try:
            if len(parameters) == 0:
                res = await function()
            elif len(parameters) == len(args) + if_self_state:
                res = await function(*args)
            elif len(parameters) == len(kwargs.keys()) + if_self_state:
                res = await function(**kwargs)
            else:
                res = await function(*args, **kwargs)
            self.logger.info(f"Execution done in {time.perf_counter()-t0:.4f}")
            if isinstance(res, Result):
                formatted_result = res
                if formatted_result.origin is None:
                    formatted_result.set_origin(mod_function_name)
            elif isinstance(res, ApiResult):
                formatted_result = res
                if formatted_result.origin is None:
                    formatted_result.as_result().set_origin(mod_function_name).to_api_result()
            elif row:
                formatted_result = res
            else:
                # Wrap the result in a Result object
                formatted_result = Result.ok(
                    interface=self.interface_type,
                    data_info="Auto generated result",
                    data=res,
                    info="Function executed successfully"
                ).set_origin(mod_function_name)
            if not row:
                self.logger.info(
                    f"Function Exec code: {formatted_result.info.exec_code} Info's: {formatted_result.info.help_text}")
            else:
                self.logger.info(
                    f"Function Exec data: {formatted_result}")
        except Exception as e:
            self.logger.error(
                Style.YELLOW(Style.Bold(
                    f"! Function ERROR: in {modular_name}.{function_name}")))
            # Wrap the exception in a Result object
            formatted_result = Result.default_internal_error(info=str(e)).set_origin(mod_function_name)
            # res = formatted_result
            self.logger.error(
                f"Function {modular_name}.{function_name}"
                f" executed wit an error {str(e)}, {type(e)}")
            self.debug_rains(e)

        else:
            self.print_ok()

            self.logger.info(
                f"Function {modular_name}.{function_name}"
                f" executed successfully")

        return formatted_result

    async def run_http(self, mod_function_name: Enum or str or tuple, function_name=None,
                       args_=None,
                       kwargs_=None, method="GET",
                       *args, **kwargs):
        if kwargs_ is not None and not kwargs:
            kwargs = kwargs_
        if args_ is not None and not args:
            args = args_

        modular_name = mod_function_name
        function_name = function_name

        if isinstance(mod_function_name, str) and isinstance(function_name, str):
            mod_function_name = (mod_function_name, function_name)

        if isinstance(mod_function_name, tuple):
            modular_name, function_name = mod_function_name
        elif isinstance(mod_function_name, list):
            modular_name, function_name = mod_function_name[0], mod_function_name[1]
        elif isinstance(mod_function_name, Enum):
            modular_name, function_name = mod_function_name.__class__.NAME.value, mod_function_name.value

        self.logger.info(f"getting function : {modular_name}.{function_name} from http {self.session.base}")
        r = await self.session.fetch(f"/api/{modular_name}/{function_name}{'?' + args_ if args_ is not None else ''}",
                                     data=kwargs, method=method)
        try:
            if not r:
                print("§ Session server Offline!", self.session.base)
                return Result.default_internal_error(info="Session fetch failed").as_dict()

            content_type = r.headers.get('Content-Type', '').lower()

            if 'application/json' in content_type:
                try:
                    return r.json()
                except Exception as e:
                    print(f"⚠ JSON decode error: {e}")
                    # Fallback to text if JSON decoding fails
                    text = r.text
            else:
                text = r.text

            if isinstance(text, Callable):
                if asyncio.iscoroutinefunction(text):
                    text = await text()
                else:
                    text = text()

            # Attempt YAML
            if 'yaml' in content_type or text.strip().startswith('---'):
                try:
                    import yaml
                    return yaml.safe_load(text)
                except Exception as e:
                    print(f"⚠ YAML decode error: {e}")

            # Attempt XML
            if 'xml' in content_type or text.strip().startswith('<?xml'):
                try:
                    import xmltodict
                    return xmltodict.parse(text)
                except Exception as e:
                    print(f"⚠ XML decode error: {e}")

            # Fallback: return plain text
            return Result.default_internal_error(data={'raw_text': text, 'content_type': content_type}).as_dict()

        except Exception as e:
            print("❌ Fatal error during API call:", e)
            self.debug_rains(e)
            return Result.default_internal_error(str(e)).as_dict()

    def run_local(self, *args, **kwargs):
        return self.run_any(*args, **kwargs)

    async def a_run_local(self, *args, **kwargs):
        return await self.a_run_any(*args, **kwargs)

    def run_any(self, mod_function_name: Enum or str or tuple, backwords_compability_variabel_string_holder=None,
                get_results=False, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None,
                kwargs_=None,
                *args, **kwargs):

        # if self.debug:
        #     self.logger.info(f'Called from: {getouterframes(currentframe(), 2)}')

        if kwargs_ is not None and not kwargs:
            kwargs = kwargs_
        if args_ is not None and not args:
            args = args_

        if isinstance(mod_function_name, str) and backwords_compability_variabel_string_holder is None:
            backwords_compability_variabel_string_holder = mod_function_name.split('.')[-1]
            mod_function_name = mod_function_name.replace(f".{backwords_compability_variabel_string_holder}", "")

        if isinstance(mod_function_name, str) and isinstance(backwords_compability_variabel_string_holder, str):
            mod_function_name = (mod_function_name, backwords_compability_variabel_string_holder)

        res: Result = self.run_function(mod_function_name,
                                        tb_run_function_with_state=tb_run_function_with_state,
                                        tb_run_with_specification=tb_run_with_specification,
                                        args_=args, kwargs_=kwargs).as_result()
        if isinstance(res, ApiResult):
            res = res.as_result()

        if isinstance(res, Result) and res.bg_task is not None:
            self.run_bg_task(res.bg_task)

        if self.debug:
            res.log(show_data=False)

        if not get_results and isinstance(res, Result):
            return res.get()

        if get_results and not isinstance(res, Result):
            return Result.ok(data=res)

        return res

    async def a_run_any(self, mod_function_name: Enum or str or tuple,
                        backwords_compability_variabel_string_holder=None,
                        get_results=False, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None,
                        kwargs_=None,
                        *args, **kwargs):

        # if self.debug:
        #     self.logger.info(f'Called from: {getouterframes(currentframe(), 2)}')

        if kwargs_ is not None and not kwargs:
            kwargs = kwargs_
        if args_ is not None and not args:
            args = args_

        if isinstance(mod_function_name, str) and backwords_compability_variabel_string_holder is None:
            backwords_compability_variabel_string_holder = mod_function_name.split('.')[-1]
            mod_function_name = mod_function_name.replace(f".{backwords_compability_variabel_string_holder}", "")

        if isinstance(mod_function_name, str) and isinstance(backwords_compability_variabel_string_holder, str):
            mod_function_name = (mod_function_name, backwords_compability_variabel_string_holder)

        res: Result = await self.a_run_function(mod_function_name,
                                                tb_run_function_with_state=tb_run_function_with_state,
                                                tb_run_with_specification=tb_run_with_specification,
                                                args_=args, kwargs_=kwargs)
        if isinstance(res, ApiResult):
            res = res.as_result()

        if isinstance(res, Result) and res.bg_task is not None:
            self.run_bg_task(res.bg_task)

        if self.debug:
            res.print()
            res.log(show_data=False) if isinstance(res, Result) else self.logger.debug(res)
        if not get_results and isinstance(res, Result):
            return res.get()

        if get_results and not isinstance(res, Result):
            return Result.ok(data=res)

        return res


    def web_context(self):
        if self._web_context is None:
            try:
                self._web_context = open("./dist/helper.html", encoding="utf-8").read()
            except Exception as e:
                self.logger.error(f"Could not load web context: {e}")
                self._web_context = "<div><h1>Web Context not found</h1></div>"
        return self._web_context

    def get_mod(self, name, spec='app') -> ModuleType or MainToolType:
        if spec != "app":
            self.print(f"Getting Module {name} spec: {spec}")
        if name not in self.functions:
            mod = self.save_load(name, spec=spec)
            if mod is False or (isinstance(mod, Result) and mod.is_error()):
                self.logger.warning(f"Could not find {name} in {list(self.functions.keys())}")
                raise ValueError(f"Could not find {name} in {list(self.functions.keys())} pleas install the module, or its posibly broken use --debug for infos")
        # private = self.functions[name].get(f"{spec}_private")
        # if private is not None:
        #     if private and spec != 'app':
        #         raise ValueError("Module is private")
        if name not in self.functions:
            self.logger.warning(f"Module '{name}' is not found")
            return None
        instance = self.functions[name].get(f"{spec}_instance")
        if instance is None:
            return self.load_mod(name, spec=spec)
        return self.functions[name].get(f"{spec}_instance")

    def print(self, text="", *args, **kwargs):
        # self.logger.info(f"Output : {text}")
        if 'live' in self.id:
            return

        flush = kwargs.pop('flush', True)
        if self.sprint(None):
            print(Style.CYAN(f"System${self.id}:"), end=" ", flush=flush)
        print(text, *args, **kwargs, flush=flush)

    def sprint(self, text="", show_system=True, *args, **kwargs):
        if text is None:
            return True
        if 'live' in self.id:
            return
        flush = kwargs.pop('flush', True)
        # self.logger.info(f"Output : {text}")
        if show_system:
            print(Style.CYAN(f"System${self.id}:"), end=" ", flush=flush)
        if isinstance(text, str) and kwargs == {} and text:
            stram_print(text + ' '.join(args))
            print()
        else:
            print(text, *args, **kwargs, flush=flush)

    # ----------------------------------------------------------------
    # Decorators for the toolbox

    def reload_mod(self, mod_name, spec='app', is_file=True, loc="toolboxv2.mods."):
        self.remove_mod(mod_name, delete=True)
        if mod_name not in self.modules:
            self.logger.warning(f"Module '{mod_name}' is not found")
            return
        if hasattr(self.modules[mod_name], 'reload_save') and self.modules[mod_name].reload_save:
            def reexecute_module_code(x):
                return x
        else:
            def reexecute_module_code(module_name):
                if isinstance(module_name, str):
                    module = import_module(module_name)
                else:
                    module = module_name
                # Get the source code of the module
                try:
                    source = inspect.getsource(module)
                except Exception:
                    # print(f"No source for {str(module_name).split('from')[0]}: {e}")
                    return module
                # Compile the source code
                try:
                    code = compile(source, module.__file__, 'exec')
                    # Execute the code in the module's namespace
                    exec(code, module.__dict__)
                except Exception:
                    # print(f"No source for {str(module_name).split('from')[0]}: {e}")
                    pass
                return module

        if not is_file:
            mods = self.get_all_mods("./mods/" + mod_name)
            def recursive_reload(package_name):
                package = import_module(package_name)

                # First, reload all submodules
                if hasattr(package, '__path__'):
                    for _finder, name, _ispkg in pkgutil.walk_packages(package.__path__, package.__name__ + "."):
                        try:
                            mod = import_module(name)
                            reexecute_module_code(mod)
                            reload(mod)
                        except Exception as e:
                            print(f"Error reloading module {name}: {e}")
                            break

                # Finally, reload the package itself
                reexecute_module_code(package)
                reload(package)

            for mod in mods:
                if mod.endswith(".txt") or mod.endswith(".yaml"):
                    continue
                try:
                    recursive_reload(loc + mod_name + '.' + mod)
                    self.print(f"Reloaded {mod_name}.{mod}")
                except ImportError:
                    self.print(f"Could not load {mod_name}.{mod}")
        reexecute_module_code(self.modules[mod_name])
        if mod_name in self.functions:
            if "on_exit" in self.functions[mod_name]:
                self.functions[mod_name]["on_exit"] = []
            if "on_start" in self.functions[mod_name]:
                self.functions[mod_name]["on_start"] = []
        self.inplace_load_instance(mod_name, spec=spec, mfo=reload(self.modules[mod_name]) if mod_name in self.modules else None)

    def watch_mod(self, mod_name, spec='app', loc="toolboxv2.mods.", use_thread=True, path_name=None, on_reload=None):
        if path_name is None:
            path_name = mod_name
        is_file = os.path.isfile(self.start_dir + '/mods/' + path_name + '.py')
        import watchfiles
        def helper():
            paths = f'mods/{path_name}' + ('.py' if is_file else '')
            self.logger.info(f'Watching Path: {paths}')
            try:
                for changes in watchfiles.watch(paths):
                    if not changes:
                        continue
                    self.reload_mod(mod_name, spec, is_file, loc)
                    if on_reload:
                        on_reload()
            except FileNotFoundError:
                self.logger.warning(f"Path {paths} not found")

        if not use_thread:
            helper()
        else:
            threading.Thread(target=helper, daemon=True).start()

    def _register_function(self, module_name, func_name, data):
        if module_name not in self.functions:
            self.functions[module_name] = {}
        if func_name in self.functions[module_name]:
            self.print(f"Overriding function {func_name} from {module_name}", end="\r")
            self.functions[module_name][func_name] = data
        else:
            self.functions[module_name][func_name] = data

    def _create_decorator(self, type_: str,
                          name: str = "",
                          mod_name: str = "",
                          level: int = -1,
                          restrict_in_virtual_mode: bool = False,
                          api: bool = False,
                          helper: str = "",
                          version: str or None = None,
                          initial: bool=False,
                          exit_f: bool=False,
                          test: bool=True,
                          samples:list[dict[str, Any]] | None=None,
                          state:bool | None=None,
                          pre_compute:Callable | None=None,
                          post_compute:Callable[[], Result] | None=None,
                          api_methods:list[str] | None=None,
                          memory_cache: bool=False,
                          file_cache: bool=False,
                          request_as_kwarg: bool=False,
                          row: bool=False,
                          memory_cache_max_size:int=100,
                          memory_cache_ttl:int=300,
                          websocket_handler: str | None = None,
                          ):

        if isinstance(type_, Enum):
            type_ = type_.value

        if memory_cache and file_cache:
            raise ValueError("Don't use both cash at the same time for the same fuction")

        use_cache = memory_cache or file_cache
        cache = {}
        if file_cache:
            cache = FileCache(folder=self.data_dir + f'\\cache\\{mod_name}\\',
                              filename=self.data_dir + f'\\cache\\{mod_name}\\{name}cache.db')
        if memory_cache:
            cache = MemoryCache(maxsize=memory_cache_max_size, ttl=memory_cache_ttl)

        version = self.version if version is None else self.version + ':' + version

        def a_additional_process(func):

            async def executor(*args, **kwargs):

                if pre_compute is not None:
                    args, kwargs = await pre_compute(*args, **kwargs)
                if asyncio.iscoroutinefunction(func):
                    result = await func(*args, **kwargs)
                else:
                    result = func(*args, **kwargs)
                if post_compute is not None:
                    result = await post_compute(result)
                if row:
                    return result
                if not isinstance(result, Result):
                    result = Result.ok(data=result)
                if result.origin is None:
                    result.set_origin((mod_name if mod_name else func.__module__.split('.')[-1]
                                       , name if name else func.__name__
                                       , type_))
                if result.result.data_to == ToolBoxInterfaces.native.name:
                    result.result.data_to = ToolBoxInterfaces.remote if api else ToolBoxInterfaces.native
                # Wenden Sie die to_api_result Methode auf das Ergebnis an, falls verfügbar
                if api and hasattr(result, 'to_api_result'):
                    return result.to_api_result()
                return result

            @wraps(func)
            async def wrapper(*args, **kwargs):

                if not use_cache:
                    return await executor(*args, **kwargs)

                try:
                    cache_key = (f"{mod_name if mod_name else func.__module__.split('.')[-1]}"
                                 f"-{func.__name__}-{str(args)},{str(kwargs.items())}")
                except ValueError:
                    cache_key = (f"{mod_name if mod_name else func.__module__.split('.')[-1]}"
                                 f"-{func.__name__}-{bytes(args)},{str(kwargs.items())}")

                result = cache.get(cache_key)
                if result is not None:
                    return result

                result = await executor(*args, **kwargs)

                cache.set(cache_key, result)

                return result

            return wrapper

        def additional_process(func):

            def executor(*args, **kwargs):

                if pre_compute is not None:
                    args, kwargs = pre_compute(*args, **kwargs)
                if asyncio.iscoroutinefunction(func):
                    result = func(*args, **kwargs)
                else:
                    result = func(*args, **kwargs)
                if post_compute is not None:
                    result = post_compute(result)
                if row:
                    return result
                if not isinstance(result, Result):
                    result = Result.ok(data=result)
                if result.origin is None:
                    result.set_origin((mod_name if mod_name else func.__module__.split('.')[-1]
                                       , name if name else func.__name__
                                       , type_))
                if result.result.data_to == ToolBoxInterfaces.native.name:
                    result.result.data_to = ToolBoxInterfaces.remote if api else ToolBoxInterfaces.native
                # Wenden Sie die to_api_result Methode auf das Ergebnis an, falls verfügbar
                if api and hasattr(result, 'to_api_result'):
                    return result.to_api_result()
                return result

            @wraps(func)
            def wrapper(*args, **kwargs):

                if not use_cache:
                    return executor(*args, **kwargs)

                try:
                    cache_key = (f"{mod_name if mod_name else func.__module__.split('.')[-1]}"
                                 f"-{func.__name__}-{str(args)},{str(kwargs.items())}")
                except ValueError:
                    cache_key = (f"{mod_name if mod_name else func.__module__.split('.')[-1]}"
                                 f"-{func.__name__}-{bytes(args)},{str(kwargs.items())}")

                result = cache.get(cache_key)
                if result is not None:
                    return result

                result = executor(*args, **kwargs)

                cache.set(cache_key, result)

                return result

            return wrapper

        def decorator(func):
            sig = signature(func)
            params = list(sig.parameters)
            module_name = mod_name if mod_name else func.__module__.split('.')[-1]
            func_name = name if name else func.__name__
            if func_name == 'on_start':
                func_name = 'on_startup'
            if func_name == 'on_exit':
                func_name = 'on_close'
            if api or pre_compute is not None or post_compute is not None or memory_cache or file_cache:
                if asyncio.iscoroutinefunction(func):
                    func = a_additional_process(func)
                else:
                    func = additional_process(func)
            if api and str(sig.return_annotation) == 'Result':
                raise ValueError(f"Fuction {module_name}.{func_name} registered as "
                                 f"Api fuction but uses {str(sig.return_annotation)}\n"
                                 f"Please change the sig from ..)-> Result to ..)-> ApiResult")
            data = {
                "type": type_,
                "module_name": module_name,
                "func_name": func_name,
                "level": level,
                "restrict_in_virtual_mode": restrict_in_virtual_mode,
                "func": func,
                "api": api,
                "helper": helper,
                "version": version,
                "initial": initial,
                "exit_f": exit_f,
                "api_methods": api_methods if api_methods is not None else ["AUTO"],
                "__module__": func.__module__,
                "signature": sig,
                "params": params,
                "row": row,
                "state": (
                    False if len(params) == 0 else params[0] in ['self', 'state', 'app']) if state is None else state,
                "do_test": test,
                "samples": samples,
                "request_as_kwarg": request_as_kwarg,

            }

            if websocket_handler:
                # Die dekorierte Funktion sollte ein Dict mit den Handlern zurückgeben
                try:
                    handler_config = func(self)  # Rufe die Funktion auf, um die Konfiguration zu erhalten
                    if not isinstance(handler_config, dict):
                        raise TypeError(
                            f"WebSocket handler function '{func.__name__}' must return a dictionary of handlers.")

                    # Handler-Identifikator, z.B. "ChatModule/room_chat"
                    handler_id = f"{module_name}/{websocket_handler}"
                    self.websocket_handlers[handler_id] = {}

                    for event_name, handler_func in handler_config.items():
                        if event_name in ["on_connect", "on_message", "on_disconnect"] and callable(handler_func):
                            self.websocket_handlers[handler_id][event_name] = handler_func
                        else:
                            self.logger.warning(f"Invalid WebSocket handler event '{event_name}' in '{handler_id}'.")

                    self.logger.info(f"Registered WebSocket handlers for '{handler_id}'.")

                except Exception as e:
                    self.logger.error(f"Failed to register WebSocket handlers for '{func.__name__}': {e}",
                                      exc_info=True)
            else:
                self._register_function(module_name, func_name, data)

            if exit_f:
                if "on_exit" not in self.functions[module_name]:
                    self.functions[module_name]["on_exit"] = []
                self.functions[module_name]["on_exit"].append(func_name)
            if initial:
                if "on_start" not in self.functions[module_name]:
                    self.functions[module_name]["on_start"] = []
                self.functions[module_name]["on_start"].append(func_name)

            return func

        decorator.tb_init = True

        return decorator

    def export(self, *args, **kwargs):
        return self.tb(*args, **kwargs)

    def tb(self, name=None,
           mod_name: str = "",
           helper: str = "",
           version: str | None = None,
           test: bool = True,
           restrict_in_virtual_mode: bool = False,
           api: bool = False,
           initial: bool = False,
           exit_f: bool = False,
           test_only: bool = False,
           memory_cache: bool = False,
           file_cache: bool = False,
           request_as_kwarg: bool = False,
           row: bool = False,
           state: bool | None = None,
           level: int = -1,
           memory_cache_max_size: int = 100,
           memory_cache_ttl: int = 300,
           samples: list or dict or None = None,
           interface: ToolBoxInterfaces or None or str = None,
           pre_compute=None,
           post_compute=None,
           api_methods=None,
           websocket_handler: str | None = None,
           ):
        """
    A decorator for registering and configuring functions within a module.

    This decorator is used to wrap functions with additional functionality such as caching, API conversion, and lifecycle management (initialization and exit). It also handles the registration of the function in the module's function registry.

    Args:
        name (str, optional): The name to register the function under. Defaults to the function's own name.
        mod_name (str, optional): The name of the module the function belongs to.
        helper (str, optional): A helper string providing additional information about the function.
        version (str or None, optional): The version of the function or module.
        test (bool, optional): Flag to indicate if the function is for testing purposes.
        restrict_in_virtual_mode (bool, optional): Flag to restrict the function in virtual mode.
        api (bool, optional): Flag to indicate if the function is part of an API.
        initial (bool, optional): Flag to indicate if the function should be executed at initialization.
        exit_f (bool, optional): Flag to indicate if the function should be executed at exit.
        test_only (bool, optional): Flag to indicate if the function should only be used for testing.
        memory_cache (bool, optional): Flag to enable memory caching for the function.
        request_as_kwarg (bool, optional): Flag to get request if the fuction is calld from api.
        file_cache (bool, optional): Flag to enable file caching for the function.
        row (bool, optional): rather to auto wrap the result in Result type default False means no row data aka result type
        state (bool or None, optional): Flag to indicate if the function maintains state.
        level (int, optional): The level of the function, used for prioritization or categorization.
        memory_cache_max_size (int, optional): Maximum size of the memory cache.
        memory_cache_ttl (int, optional): Time-to-live for the memory cache entries.
        samples (list or dict or None, optional): Samples or examples of function usage.
        interface (str, optional): The interface type for the function.
        pre_compute (callable, optional): A function to be called before the main function.
        post_compute (callable, optional): A function to be called after the main function.
        api_methods (list[str], optional): default ["AUTO"] (GET if not params, POST if params) , GET, POST, PUT or DELETE.
        websocket_handler (str, optional): The name of the websocket handler to use.

    Returns:
        function: The decorated function with additional processing and registration capabilities.
    """
        if interface is None:
            interface = "tb"
        if test_only and 'test' not in self.id:
            return lambda *args, **kwargs: args
        return self._create_decorator(interface,
                                      name,
                                      mod_name,
                                      level=level,
                                      restrict_in_virtual_mode=restrict_in_virtual_mode,
                                      helper=helper,
                                      api=api,
                                      version=version,
                                      initial=initial,
                                      exit_f=exit_f,
                                      test=test,
                                      samples=samples,
                                      state=state,
                                      pre_compute=pre_compute,
                                      post_compute=post_compute,
                                      memory_cache=memory_cache,
                                      file_cache=file_cache,
                                      request_as_kwarg=request_as_kwarg,
                                      row=row,
                                      api_methods=api_methods,
                                      memory_cache_max_size=memory_cache_max_size,
                                      memory_cache_ttl=memory_cache_ttl,
                                      websocket_handler=websocket_handler,
                                      )

    def save_autocompletion_dict(self):
        autocompletion_dict = {}
        for module_name, _module in self.functions.items():
            data = {}
            for function_name, function_data in self.functions[module_name].items():
                if not isinstance(function_data, dict):
                    continue
                data[function_name] = {arg: None for arg in
                                       function_data.get("params", [])}
                if len(data[function_name].keys()) == 0:
                    data[function_name] = None
            autocompletion_dict[module_name] = data if len(data.keys()) > 0 else None
        self.config_fh.add_to_save_file_handler("auto~~~~~~", str(autocompletion_dict))

    def get_autocompletion_dict(self):
        return self.config_fh.get_file_handler("auto~~~~~~")

    def save_registry_as_enums(self, directory: str, filename: str):
        # Ordner erstellen, falls nicht vorhanden
        if not os.path.exists(directory):
            os.makedirs(directory)

        # Dateipfad vorbereiten
        filepath = os.path.join(directory, filename)

        # Enum-Klassen als Strings generieren
        enum_classes = [f'"""Automatic generated by ToolBox v = {self.version}"""'
                        f'\nfrom enum import Enum\nfrom dataclasses import dataclass'
                        f'\n\n\n']
        for module, functions in self.functions.items():
            if module.startswith("APP_INSTANCE"):
                continue
            class_name = module
            enum_members = "\n    ".join(
                [
                    f"{func_name.upper().replace('-', '')}"
                    f" = '{func_name}' "
                    f"# Input: ({fuction_data['params'] if isinstance(fuction_data, dict) else ''}),"
                    f" Output: {fuction_data['signature'].return_annotation if isinstance(fuction_data, dict) else 'None'}"
                    for func_name, fuction_data in functions.items()])
            enum_class = (f'@dataclass\nclass {class_name.upper().replace(".", "_").replace("-", "")}(Enum):'
                          f"\n    NAME = '{class_name}'\n    {enum_members}")
            enum_classes.append(enum_class)

        # Enums in die Datei schreiben
        data = "\n\n\n".join(enum_classes)
        if len(data) < 12:
            raise ValueError(
                "Invalid Enums Loosing content pleas delete it ur self in the (utils/system/all_functions_enums.py) or add mor new stuff :}")
        with open(filepath, 'w') as file:
            file.write(data)

        print(Style.Bold(Style.BLUE(f"Enums gespeichert in {filepath}")))


    # WS logic

    def _set_rust_ws_bridge(self, bridge_object: Any):
        """
        Diese Methode wird von Rust aufgerufen, um die Kommunikationsbrücke zu setzen.
        Sie darf NICHT manuell von Python aus aufgerufen werden.
        """
        self.print(f"Rust WebSocket bridge has been set for instance {self.id}.")
        self._rust_ws_bridge = bridge_object

    async def ws_send(self, conn_id: str, payload: dict):
        """
        Sendet eine Nachricht asynchron an eine einzelne WebSocket-Verbindung.

        Args:
            conn_id: Die eindeutige ID der Zielverbindung.
            payload: Ein Dictionary, das als JSON gesendet wird.
        """
        if self._rust_ws_bridge is None:
            self.logger.error("Cannot send WebSocket message: Rust bridge is not initialized.")
            return

        try:
            # Ruft die asynchrone Rust-Methode auf und wartet auf deren Abschluss
            await self._rust_ws_bridge.send_message(conn_id, json.dumps(payload))
        except Exception as e:
            self.logger.error(f"Failed to send WebSocket message to {conn_id}: {e}", exc_info=True)

    async def ws_broadcast(self, channel_id: str, payload: dict, source_conn_id: str = "python_broadcast"):
        """
        Sendet eine Nachricht asynchron an alle Clients in einem Kanal/Raum.

        Args:
            channel_id: Der Kanal, an den gesendet werden soll.
            payload: Ein Dictionary, das als JSON gesendet wird.
            source_conn_id (optional): Die ID der ursprünglichen Verbindung, um Echos zu vermeiden.
        """
        if self._rust_ws_bridge is None:
            self.logger.error("Cannot broadcast WebSocket message: Rust bridge is not initialized.")
            return

        try:
            # Ruft die asynchrone Rust-Broadcast-Methode auf
            await self._rust_ws_bridge.broadcast_message(channel_id, json.dumps(payload), source_conn_id)
        except Exception as e:
            self.logger.error(f"Failed to broadcast WebSocket message to channel {channel_id}: {e}", exc_info=True)

disconnect(*args, **kwargs) staticmethod

proxi attr

Source code in toolboxv2/utils/toolbox.py

@staticmethod
def disconnect(*args, **kwargs):
    """proxi attr"""

exit_main(*args, **kwargs) staticmethod

proxi attr

Source code in toolboxv2/utils/toolbox.py

@staticmethod
def exit_main(*args, **kwargs):
    """proxi attr"""

get_function(name, **kwargs)

Kwargs for _get_function metadata:: return the registered function dictionary stateless: (function_data, None), 0 stateful: (function_data, higher_order_function), 0 state::boolean specification::str default app

Source code in toolboxv2/utils/toolbox.py

def get_function(self, name: Enum or tuple, **kwargs):
    """
    Kwargs for _get_function
        metadata:: return the registered function dictionary
            stateless: (function_data, None), 0
            stateful: (function_data, higher_order_function), 0
        state::boolean
            specification::str default app
    """
    if isinstance(name, tuple):
        return self._get_function(None, as_str=name, **kwargs)
    else:
        return self._get_function(name, **kwargs)

hide_console(*args, **kwargs) staticmethod

proxi attr

Source code in toolboxv2/utils/toolbox.py

@staticmethod
def hide_console(*args, **kwargs):
    """proxi attr"""

init_mod(mod_name, spec='app')

Initializes a module in a thread-safe manner by submitting the asynchronous initialization to the running event loop.

Source code in toolboxv2/utils/toolbox.py

def init_mod(self, mod_name, spec='app'):
    """
    Initializes a module in a thread-safe manner by submitting the
    asynchronous initialization to the running event loop.
    """
    if '.' in mod_name:
        mod_name = mod_name.split('.')[0]
    self.run_bg_task(self.a_init_mod, mod_name, spec)

run(*args, request=None, running_function_coro=None, **kwargs)

Run a function with support for SSE streaming in both threaded and non-threaded contexts.

Source code in toolboxv2/utils/toolbox.py

def run(self, *args, request=None, running_function_coro=None, **kwargs):
    """
    Run a function with support for SSE streaming in both
    threaded and non-threaded contexts.
    """
    if running_function_coro is None:
        mn, fn = args[0]
        if self.functions.get(mn, {}).get(fn, {}).get('request_as_kwarg', False):
            kwargs["request"] = RequestData.from_dict(request)
            if 'data' in kwargs and 'data' not in self.functions.get(mn, {}).get(fn, {}).get('params', []):
                kwargs["request"].data = kwargs["request"].body = kwargs['data']
                del kwargs['data']
            if 'form_data' in kwargs and 'form_data' not in self.functions.get(mn, {}).get(fn, {}).get('params',
                                                                                                       []):
                kwargs["request"].form_data = kwargs["request"].body = kwargs['form_data']
                del kwargs['form_data']
        else:
            params = self.functions.get(mn, {}).get(fn, {}).get('params', [])
            # auto pars data and form_data to kwargs by key
            do = False
            data = {}
            if 'data' in kwargs and 'data' not in params:
                do = True
                data = kwargs['data']
                del kwargs['data']
            if 'form_data' in kwargs and 'form_data' not in params:
                do = True
                data = kwargs['form_data']
                del kwargs['form_data']
            if do:
                for k in params:
                    if k in data:
                        kwargs[k] = data[k]
                        del data[k]

    # Create the coroutine
    coro = running_function_coro or self.a_run_any(*args, **kwargs)

    # Get or create an event loop
    try:
        loop = asyncio.get_event_loop()
        is_running = loop.is_running()
    except RuntimeError:
        loop = asyncio.new_event_loop()
        asyncio.set_event_loop(loop)
        is_running = False

    # If the loop is already running, run in a separate thread
    if is_running:
        # Create thread pool executor as needed
        if not hasattr(self.__class__, '_executor'):
            self.__class__._executor = ThreadPoolExecutor(max_workers=4)

        def run_in_new_thread():
            # Set up a new loop in this thread
            new_loop = asyncio.new_event_loop()
            asyncio.set_event_loop(new_loop)

            try:
                # Run the coroutine
                return new_loop.run_until_complete(coro)
            finally:
                new_loop.close()

        # Run in thread and get result
        thread_result = self.__class__._executor.submit(run_in_new_thread).result()

        # Handle streaming results from thread
        if isinstance(thread_result, dict) and thread_result.get("is_stream"):
            # Create a new SSE stream in the main thread
            async def stream_from_function():
                # Re-run the function with direct async access
                stream_result = await self.a_run_any(*args, **kwargs)

                if (isinstance(stream_result, Result) and
                    getattr(stream_result.result, 'data_type', None) == "stream"):
                    # Get and forward data from the original generator
                    original_gen = stream_result.result.data.get("generator")
                    if inspect.isasyncgen(original_gen):
                        async for item in original_gen:
                            yield item

            # Return a new streaming Result
            return Result.stream(
                stream_generator=stream_from_function(),
                headers=thread_result.get("headers", {})
            )

        result = thread_result
    else:
        # Direct execution when loop is not running
        result = loop.run_until_complete(coro)

    # Process the final result
    if isinstance(result, Result):
        if 'debug' in self.id:
            result.print()
        if getattr(result.result, 'data_type', None) == "stream":
            return result
        return result.to_api_result().model_dump(mode='json')

    return result

run_bg_task(task, *args, **kwargs)

Runs a coroutine in the background without blocking the caller.

This is the primary method for "fire-and-forget" async tasks. It schedules the coroutine to run on the application's main event loop.

Parameters:

Name	Type	Description	Default
task	Callable	The coroutine function to run.	required
*args		Arguments to pass to the coroutine function.	()
**kwargs		Keyword arguments to pass to the coroutine function.	{}

Returns:

Type	Description
Task | None	An asyncio.Task object representing the scheduled task, or None if
Task | None	the task could not be scheduled.

Source code in toolboxv2/utils/toolbox.py

def run_bg_task(self, task: Callable, *args, **kwargs) -> asyncio.Task | None:
    """
    Runs a coroutine in the background without blocking the caller.

    This is the primary method for "fire-and-forget" async tasks. It schedules
    the coroutine to run on the application's main event loop.

    Args:
        task: The coroutine function to run.
        *args: Arguments to pass to the coroutine function.
        **kwargs: Keyword arguments to pass to the coroutine function.

    Returns:
        An asyncio.Task object representing the scheduled task, or None if
        the task could not be scheduled.
    """
    if not callable(task):
        self.logger.warning("Task passed to run_bg_task is not callable!")
        return None

    if not asyncio.iscoroutinefunction(task) and not asyncio.iscoroutine(task):
        self.logger.warning(f"Task '{getattr(task, '__name__', 'unknown')}' is not a coroutine. "
                            f"Use run_bg_task_advanced for synchronous functions.")
        # Fallback to advanced runner for convenience
        self.run_bg_task_advanced(task, *args, **kwargs)
        return None

    try:
        loop = self.loop_gard()
        if not loop.is_running():
            # If the main loop isn't running, we can't create a task on it.
            # This scenario is handled by run_bg_task_advanced.
            self.logger.info("Main event loop not running. Delegating to advanced background runner.")
            return self.run_bg_task_advanced(task, *args, **kwargs)

        # Create the coroutine if it's a function
        coro = task(*args, **kwargs) if asyncio.iscoroutinefunction(task) else task

        # Create a task on the running event loop
        bg_task = loop.create_task(coro)

        # Add a callback to log exceptions from the background task
        def _log_exception(the_task: asyncio.Task):
            if not the_task.cancelled() and the_task.exception():
                self.logger.error(f"Exception in background task '{the_task.get_name()}':",
                                  exc_info=the_task.exception())

        bg_task.add_done_callback(_log_exception)
        self.bg_tasks.append(bg_task)
        return bg_task

    except Exception as e:
        self.logger.error(f"Failed to schedule background task: {e}", exc_info=True)
        return None

run_bg_task_advanced(task, *args, **kwargs)

Runs a task in a separate, dedicated background thread with its own event loop.

This is ideal for: 1. Running an async task from a synchronous context. 2. Launching a long-running, independent operation that should not interfere with the main application's event loop.

Parameters:

Name	Type	Description	Default
task	Callable	The function to run (can be sync or async).	required
*args		Arguments for the task.	()
**kwargs		Keyword arguments for the task.	{}

Returns:

Type	Description
Thread	The threading.Thread object managing the background execution.

Source code in toolboxv2/utils/toolbox.py

def run_bg_task_advanced(self, task: Callable, *args, **kwargs) -> threading.Thread:
    """
    Runs a task in a separate, dedicated background thread with its own event loop.

    This is ideal for:
    1. Running an async task from a synchronous context.
    2. Launching a long-running, independent operation that should not
       interfere with the main application's event loop.

    Args:
        task: The function to run (can be sync or async).
        *args: Arguments for the task.
        **kwargs: Keyword arguments for the task.

    Returns:
        The threading.Thread object managing the background execution.
    """
    if not callable(task):
        self.logger.warning("Task for run_bg_task_advanced is not callable!")
        return None

    def thread_target():
        # Each thread gets its own event loop.
        loop = asyncio.new_event_loop()
        asyncio.set_event_loop(loop)

        try:
            # Prepare the coroutine we need to run
            if asyncio.iscoroutinefunction(task):
                coro = task(*args, **kwargs)
            elif asyncio.iscoroutine(task):
                # It's already a coroutine object
                coro = task
            else:
                # It's a synchronous function, run it in an executor
                # to avoid blocking the new event loop.
                coro = loop.run_in_executor(None, lambda: task(*args, **kwargs))

            # Run the coroutine to completion
            result = loop.run_until_complete(coro)
            self.logger.debug(f"Advanced background task '{getattr(task, '__name__', 'unknown')}' completed.")
            if result is not None:
                self.logger.debug(f"Task result: {str(result)[:100]}")

        except Exception as e:
            self.logger.error(f"Error in advanced background task '{getattr(task, '__name__', 'unknown')}':",
                              exc_info=e)
        finally:
            # Cleanly shut down the event loop in this thread.
            try:
                all_tasks = asyncio.all_tasks(loop=loop)
                if all_tasks:
                    for t in all_tasks:
                        t.cancel()
                    loop.run_until_complete(asyncio.gather(*all_tasks, return_exceptions=True))
            finally:
                loop.close()
                asyncio.set_event_loop(None)

    # Create, start, and return the thread.
    # It's a daemon thread so it won't prevent the main app from exiting.
    t = threading.Thread(target=thread_target, daemon=True, name=f"BGTask-{getattr(task, '__name__', 'unknown')}")
    self.bg_tasks.append(t)
    t.start()
    return t

show_console(*args, **kwargs) staticmethod

proxi attr

Source code in toolboxv2/utils/toolbox.py

@staticmethod
def show_console(*args, **kwargs):
    """proxi attr"""

tb(name=None, mod_name='', helper='', version=None, test=True, restrict_in_virtual_mode=False, api=False, initial=False, exit_f=False, test_only=False, memory_cache=False, file_cache=False, request_as_kwarg=False, row=False, state=None, level=-1, memory_cache_max_size=100, memory_cache_ttl=300, samples=None, interface=None, pre_compute=None, post_compute=None, api_methods=None, websocket_handler=None)

A decorator for registering and configuring functions within a module.

This decorator is used to wrap functions with additional functionality such as caching, API conversion, and lifecycle management (initialization and exit). It also handles the registration of the function in the module's function registry.

Parameters:

Name	Type	Description	Default
name	str	The name to register the function under. Defaults to the function's own name.	None
mod_name	str	The name of the module the function belongs to.	''
helper	str	A helper string providing additional information about the function.	''
version	str or None	The version of the function or module.	None
test	bool	Flag to indicate if the function is for testing purposes.	True
restrict_in_virtual_mode	bool	Flag to restrict the function in virtual mode.	False
api	bool	Flag to indicate if the function is part of an API.	False
initial	bool	Flag to indicate if the function should be executed at initialization.	False
exit_f	bool	Flag to indicate if the function should be executed at exit.	False
test_only	bool	Flag to indicate if the function should only be used for testing.	False
memory_cache	bool	Flag to enable memory caching for the function.	False
request_as_kwarg	bool	Flag to get request if the fuction is calld from api.	False
file_cache	bool	Flag to enable file caching for the function.	False
row	bool	rather to auto wrap the result in Result type default False means no row data aka result type	False
state	bool or None	Flag to indicate if the function maintains state.	None
level	int	The level of the function, used for prioritization or categorization.	-1
memory_cache_max_size	int	Maximum size of the memory cache.	100
memory_cache_ttl	int	Time-to-live for the memory cache entries.	300
samples	list or dict or None	Samples or examples of function usage.	None
interface	str	The interface type for the function.	None
pre_compute	callable	A function to be called before the main function.	None
post_compute	callable	A function to be called after the main function.	None
api_methods	list[str]	default ["AUTO"] (GET if not params, POST if params) , GET, POST, PUT or DELETE.	None
websocket_handler	str	The name of the websocket handler to use.	None

Returns:

Name	Type	Description
function		The decorated function with additional processing and registration capabilities.

Source code in toolboxv2/utils/toolbox.py

def tb(self, name=None,
       mod_name: str = "",
       helper: str = "",
       version: str | None = None,
       test: bool = True,
       restrict_in_virtual_mode: bool = False,
       api: bool = False,
       initial: bool = False,
       exit_f: bool = False,
       test_only: bool = False,
       memory_cache: bool = False,
       file_cache: bool = False,
       request_as_kwarg: bool = False,
       row: bool = False,
       state: bool | None = None,
       level: int = -1,
       memory_cache_max_size: int = 100,
       memory_cache_ttl: int = 300,
       samples: list or dict or None = None,
       interface: ToolBoxInterfaces or None or str = None,
       pre_compute=None,
       post_compute=None,
       api_methods=None,
       websocket_handler: str | None = None,
       ):
    """
A decorator for registering and configuring functions within a module.

This decorator is used to wrap functions with additional functionality such as caching, API conversion, and lifecycle management (initialization and exit). It also handles the registration of the function in the module's function registry.

Args:
    name (str, optional): The name to register the function under. Defaults to the function's own name.
    mod_name (str, optional): The name of the module the function belongs to.
    helper (str, optional): A helper string providing additional information about the function.
    version (str or None, optional): The version of the function or module.
    test (bool, optional): Flag to indicate if the function is for testing purposes.
    restrict_in_virtual_mode (bool, optional): Flag to restrict the function in virtual mode.
    api (bool, optional): Flag to indicate if the function is part of an API.
    initial (bool, optional): Flag to indicate if the function should be executed at initialization.
    exit_f (bool, optional): Flag to indicate if the function should be executed at exit.
    test_only (bool, optional): Flag to indicate if the function should only be used for testing.
    memory_cache (bool, optional): Flag to enable memory caching for the function.
    request_as_kwarg (bool, optional): Flag to get request if the fuction is calld from api.
    file_cache (bool, optional): Flag to enable file caching for the function.
    row (bool, optional): rather to auto wrap the result in Result type default False means no row data aka result type
    state (bool or None, optional): Flag to indicate if the function maintains state.
    level (int, optional): The level of the function, used for prioritization or categorization.
    memory_cache_max_size (int, optional): Maximum size of the memory cache.
    memory_cache_ttl (int, optional): Time-to-live for the memory cache entries.
    samples (list or dict or None, optional): Samples or examples of function usage.
    interface (str, optional): The interface type for the function.
    pre_compute (callable, optional): A function to be called before the main function.
    post_compute (callable, optional): A function to be called after the main function.
    api_methods (list[str], optional): default ["AUTO"] (GET if not params, POST if params) , GET, POST, PUT or DELETE.
    websocket_handler (str, optional): The name of the websocket handler to use.

Returns:
    function: The decorated function with additional processing and registration capabilities.
"""
    if interface is None:
        interface = "tb"
    if test_only and 'test' not in self.id:
        return lambda *args, **kwargs: args
    return self._create_decorator(interface,
                                  name,
                                  mod_name,
                                  level=level,
                                  restrict_in_virtual_mode=restrict_in_virtual_mode,
                                  helper=helper,
                                  api=api,
                                  version=version,
                                  initial=initial,
                                  exit_f=exit_f,
                                  test=test,
                                  samples=samples,
                                  state=state,
                                  pre_compute=pre_compute,
                                  post_compute=post_compute,
                                  memory_cache=memory_cache,
                                  file_cache=file_cache,
                                  request_as_kwarg=request_as_kwarg,
                                  row=row,
                                  api_methods=api_methods,
                                  memory_cache_max_size=memory_cache_max_size,
                                  memory_cache_ttl=memory_cache_ttl,
                                  websocket_handler=websocket_handler,
                                  )

wait_for_bg_tasks(timeout=None)

Wait for all background tasks to complete.

Parameters:

Name	Type	Description	Default
timeout		Maximum time to wait (in seconds) for all tasks to complete. None means wait indefinitely.	None

Returns:

Name	Type	Description
bool		True if all tasks completed, False if timeout occurred

Source code in toolboxv2/utils/toolbox.py

def wait_for_bg_tasks(self, timeout=None):
    """
    Wait for all background tasks to complete.

    Args:
        timeout: Maximum time to wait (in seconds) for all tasks to complete.
                 None means wait indefinitely.

    Returns:
        bool: True if all tasks completed, False if timeout occurred
    """
    active_tasks = [t for t in self.bg_tasks if t.is_alive()]

    for task in active_tasks:
        task.join(timeout=timeout)
        if task.is_alive():
            return False

    return True

ws_broadcast(channel_id, payload, source_conn_id='python_broadcast') async

Sendet eine Nachricht asynchron an alle Clients in einem Kanal/Raum.

Parameters:

Name	Type	Description	Default
channel_id	str	Der Kanal, an den gesendet werden soll.	required
payload	dict	Ein Dictionary, das als JSON gesendet wird.	required
source_conn_id	optional	Die ID der ursprünglichen Verbindung, um Echos zu vermeiden.	'python_broadcast'

Source code in toolboxv2/utils/toolbox.py

async def ws_broadcast(self, channel_id: str, payload: dict, source_conn_id: str = "python_broadcast"):
    """
    Sendet eine Nachricht asynchron an alle Clients in einem Kanal/Raum.

    Args:
        channel_id: Der Kanal, an den gesendet werden soll.
        payload: Ein Dictionary, das als JSON gesendet wird.
        source_conn_id (optional): Die ID der ursprünglichen Verbindung, um Echos zu vermeiden.
    """
    if self._rust_ws_bridge is None:
        self.logger.error("Cannot broadcast WebSocket message: Rust bridge is not initialized.")
        return

    try:
        # Ruft die asynchrone Rust-Broadcast-Methode auf
        await self._rust_ws_bridge.broadcast_message(channel_id, json.dumps(payload), source_conn_id)
    except Exception as e:
        self.logger.error(f"Failed to broadcast WebSocket message to channel {channel_id}: {e}", exc_info=True)

ws_send(conn_id, payload) async

Sendet eine Nachricht asynchron an eine einzelne WebSocket-Verbindung.

Parameters:

Name	Type	Description	Default
conn_id	str	Die eindeutige ID der Zielverbindung.	required
payload	dict	Ein Dictionary, das als JSON gesendet wird.	required

Source code in toolboxv2/utils/toolbox.py

async def ws_send(self, conn_id: str, payload: dict):
    """
    Sendet eine Nachricht asynchron an eine einzelne WebSocket-Verbindung.

    Args:
        conn_id: Die eindeutige ID der Zielverbindung.
        payload: Ein Dictionary, das als JSON gesendet wird.
    """
    if self._rust_ws_bridge is None:
        self.logger.error("Cannot send WebSocket message: Rust bridge is not initialized.")
        return

    try:
        # Ruft die asynchrone Rust-Methode auf und wartet auf deren Abschluss
        await self._rust_ws_bridge.send_message(conn_id, json.dumps(payload))
    except Exception as e:
        self.logger.error(f"Failed to send WebSocket message to {conn_id}: {e}", exc_info=True)

Code

Source code in toolboxv2/utils/security/cryp.py

class Code:

    @staticmethod
    def DK():
        return DEVICE_KEY

    @staticmethod
    def generate_random_string(length: int) -> str:
        """
        Generiert eine zufällige Zeichenkette der angegebenen Länge.

        Args:
            length (int): Die Länge der zu generierenden Zeichenkette.

        Returns:
            str: Die generierte Zeichenkette.
        """
        return secrets.token_urlsafe(length)

    def decode_code(self, encrypted_data, key=None):

        if not isinstance(encrypted_data, str):
            encrypted_data = str(encrypted_data)

        if key is None:
            key = DEVICE_KEY()

        return self.decrypt_symmetric(encrypted_data, key)

    def encode_code(self, data, key=None):

        if not isinstance(data, str):
            data = str(data)

        if key is None:
            key = DEVICE_KEY()

        return self.encrypt_symmetric(data, key)

    @staticmethod
    def generate_seed() -> int:
        """
        Erzeugt eine zufällige Zahl als Seed.

        Returns:
            int: Eine zufällige Zahl.
        """
        return random.randint(2 ** 32 - 1, 2 ** 64 - 1)

    @staticmethod
    def one_way_hash(text: str, salt: str = '', pepper: str = '') -> str:
        """
        Erzeugt einen Hash eines gegebenen Textes mit Salt, Pepper und optional einem Seed.

        Args:
            text (str): Der zu hashende Text.
            salt (str): Der Salt-Wert.
            pepper (str): Der Pepper-Wert.
            seed (int, optional): Ein optionaler Seed-Wert. Standardmäßig None.

        Returns:
            str: Der resultierende Hash-Wert.
        """
        return hashlib.sha256((salt + text + pepper).encode()).hexdigest()

    @staticmethod
    def generate_symmetric_key(as_str=True) -> str or bytes:
        """
        Generiert einen Schlüssel für die symmetrische Verschlüsselung.

        Returns:
            str: Der generierte Schlüssel.
        """
        key = Fernet.generate_key()
        if as_str:
            key = key.decode()
        return key

    @staticmethod
    def encrypt_symmetric(text: str or bytes, key: str) -> str:
        """
        Verschlüsselt einen Text mit einem gegebenen symmetrischen Schlüssel.

        Args:
            text (str): Der zu verschlüsselnde Text.
            key (str): Der symmetrische Schlüssel.

        Returns:
            str: Der verschlüsselte Text.
        """
        if isinstance(text, str):
            text = text.encode()

        try:
            fernet = Fernet(key.encode())
            return fernet.encrypt(text).decode()
        except Exception as e:
            get_logger().error(f"Error encrypt_symmetric #{str(e)}#")
            return "Error encrypt"

    @staticmethod
    def decrypt_symmetric(encrypted_text: str, key: str, to_str=True, mute=False) -> str or bytes:
        """
        Entschlüsselt einen Text mit einem gegebenen symmetrischen Schlüssel.

        Args:
            encrypted_text (str): Der zu entschlüsselnde Text.
            key (str): Der symmetrische Schlüssel.
            to_str (bool): default true returns str if false returns bytes
        Returns:
            str: Der entschlüsselte Text.
        """

        if isinstance(key, str):
            key = key.encode()

        #try:
        fernet = Fernet(key)
        text_b = fernet.decrypt(encrypted_text)
        if not to_str:
            return text_b
        return text_b.decode()
        # except Exception as e:
        #     get_logger().error(f"Error decrypt_symmetric {e}")
        #     if not mute:
        #         raise e
        #     if not to_str:
        #         return f"Error decoding".encode()
        #     return f"Error decoding"

    @staticmethod
    def generate_asymmetric_keys() -> (str, str):
        """
        Generiert ein Paar von öffentlichen und privaten Schlüsseln für die asymmetrische Verschlüsselung.

        Args:
            seed (int, optional): Ein optionaler Seed-Wert. Standardmäßig None.

        Returns:
            (str, str): Ein Tupel aus öffentlichem und privatem Schlüssel.
        """
        private_key = rsa.generate_private_key(
            public_exponent=65537,
            key_size=2048 * 3,
        )
        public_key = private_key.public_key()

        # Serialisieren der Schlüssel
        pem_private_key = private_key.private_bytes(
            encoding=serialization.Encoding.PEM,
            format=serialization.PrivateFormat.PKCS8,
            encryption_algorithm=serialization.NoEncryption()
        ).decode()

        pem_public_key = public_key.public_bytes(
            encoding=serialization.Encoding.PEM,
            format=serialization.PublicFormat.SubjectPublicKeyInfo
        ).decode()

        return pem_public_key, pem_private_key

    @staticmethod
    def save_keys_to_files(public_key: str, private_key: str, directory: str = "keys") -> None:
        """
        Speichert die generierten Schlüssel in separate Dateien.
        Der private Schlüssel wird mit dem Device Key verschlüsselt.

        Args:
            public_key (str): Der öffentliche Schlüssel im PEM-Format
            private_key (str): Der private Schlüssel im PEM-Format
            directory (str): Das Verzeichnis, in dem die Schlüssel gespeichert werden sollen
        """
        # Erstelle das Verzeichnis, falls es nicht existiert
        os.makedirs(directory, exist_ok=True)

        # Hole den Device Key
        device_key = DEVICE_KEY()

        # Verschlüssele den privaten Schlüssel mit dem Device Key
        encrypted_private_key = Code.encrypt_symmetric(private_key, device_key)

        # Speichere den öffentlichen Schlüssel
        public_key_path = os.path.join(directory, "public_key.pem")
        with open(public_key_path, "w") as f:
            f.write(public_key)

        # Speichere den verschlüsselten privaten Schlüssel
        private_key_path = os.path.join(directory, "private_key.pem")
        with open(private_key_path, "w") as f:
            f.write(encrypted_private_key)

        print("Saved keys in ", public_key_path)

    @staticmethod
    def load_keys_from_files(directory: str = "keys") -> (str, str):
        """
        Lädt die Schlüssel aus den Dateien.
        Der private Schlüssel wird mit dem Device Key entschlüsselt.

        Args:
            directory (str): Das Verzeichnis, aus dem die Schlüssel geladen werden sollen

        Returns:
            (str, str): Ein Tupel aus öffentlichem und privatem Schlüssel

        Raises:
            FileNotFoundError: Wenn die Schlüsseldateien nicht gefunden werden können
        """
        # Pfade zu den Schlüsseldateien
        public_key_path = os.path.join(directory, "public_key.pem")
        private_key_path = os.path.join(directory, "private_key.pem")

        # Prüfe ob die Dateien existieren
        if not os.path.exists(public_key_path) or not os.path.exists(private_key_path):
            return "", ""

        # Hole den Device Key
        device_key = DEVICE_KEY()

        # Lade den öffentlichen Schlüssel
        with open(public_key_path) as f:
            public_key = f.read()

        # Lade und entschlüssele den privaten Schlüssel
        with open(private_key_path) as f:
            encrypted_private_key = f.read()
            private_key = Code.decrypt_symmetric(encrypted_private_key, device_key)

        return public_key, private_key

    @staticmethod
    def encrypt_asymmetric(text: str, public_key_str: str) -> str:
        """
        Verschlüsselt einen Text mit einem gegebenen öffentlichen Schlüssel.

        Args:
            text (str): Der zu verschlüsselnde Text.
            public_key_str (str): Der öffentliche Schlüssel als String oder im pem format.

        Returns:
            str: Der verschlüsselte Text.
        """
        # try:
        #    public_key: RSAPublicKey = serialization.load_pem_public_key(public_key_str.encode())
        #  except Exception as e:
        #     get_logger().error(f"Error encrypt_asymmetric {e}")
        try:
            public_key: RSAPublicKey = serialization.load_pem_public_key(public_key_str.encode())
            encrypted = public_key.encrypt(
                text.encode(),
                padding.OAEP(
                    mgf=padding.MGF1(algorithm=hashes.SHA512()),
                    algorithm=hashes.SHA512(),
                    label=None
                )
            )
            return encrypted.hex()
        except Exception as e:
            get_logger().error(f"Error encrypt_asymmetric {e}")
            return "Invalid"

    @staticmethod
    def decrypt_asymmetric(encrypted_text_hex: str, private_key_str: str) -> str:
        """
        Entschlüsselt einen Text mit einem gegebenen privaten Schlüssel.

        Args:
            encrypted_text_hex (str): Der verschlüsselte Text als Hex-String.
            private_key_str (str): Der private Schlüssel als String.

        Returns:
            str: Der entschlüsselte Text.
        """
        try:
            private_key = serialization.load_pem_private_key(private_key_str.encode(), password=None)
            decrypted = private_key.decrypt(
                bytes.fromhex(encrypted_text_hex),
                padding.OAEP(
                    mgf=padding.MGF1(algorithm=hashes.SHA512()),
                    algorithm=hashes.SHA512(),
                    label=None
                )
            )
            return decrypted.decode()

        except Exception as e:
            get_logger().error(f"Error decrypt_asymmetric {e}")
        return "Invalid"

    @staticmethod
    def verify_signature(signature: str or bytes, message: str or bytes, public_key_str: str,
                         salt_length=padding.PSS.MAX_LENGTH) -> bool:
        if isinstance(signature, str):
            signature = signature.encode()
        if isinstance(message, str):
            message = message.encode()
        try:
            public_key: RSAPublicKey = serialization.load_pem_public_key(public_key_str.encode())
            public_key.verify(
                signature=signature,
                data=message,
                padding=padding.PSS(
                    mgf=padding.MGF1(hashes.SHA512()),
                    salt_length=salt_length
                ),
                algorithm=hashes.SHA512()
            )
            return True
        except:
            pass
        return False

    @staticmethod
    def verify_signature_web_algo(signature: str or bytes, message: str or bytes, public_key_str: str,
                                  algo: int = -512) -> bool:
        signature_algorithm = ECDSA(hashes.SHA512())
        if algo != -512:
            signature_algorithm = ECDSA(hashes.SHA256())

        if isinstance(signature, str):
            signature = signature.encode()
        if isinstance(message, str):
            message = message.encode()
        try:
            public_key = serialization.load_pem_public_key(public_key_str.encode())
            public_key.verify(
                signature=signature,
                data=message,
                # padding=padding.PSS(
                #    mgf=padding.MGF1(hashes.SHA512()),
                #    salt_length=padding.PSS.MAX_LENGTH
                # ),
                signature_algorithm=signature_algorithm
            )
            return True
        except:
            pass
        return False

    @staticmethod
    def create_signature(message: str, private_key_str: str, salt_length=padding.PSS.MAX_LENGTH,
                         row=False) -> str or bytes:
        try:
            private_key = serialization.load_pem_private_key(private_key_str.encode(), password=None)
            signature = private_key.sign(
                message.encode(),
                padding.PSS(
                    mgf=padding.MGF1(hashes.SHA512()),
                    salt_length=salt_length
                ),
                hashes.SHA512()
            )
            if row:
                return signature
            return base64.b64encode(signature).decode()
        except Exception as e:
            get_logger().error(f"Error create_signature {e}")
            print(e)
        return "Invalid Key"

    @staticmethod
    def pem_to_public_key(pem_key: str):
        """
        Konvertiert einen PEM-kodierten öffentlichen Schlüssel in ein PublicKey-Objekt.

        Args:
            pem_key (str): Der PEM-kodierte öffentliche Schlüssel.

        Returns:
            PublicKey: Das PublicKey-Objekt.
        """
        public_key = serialization.load_pem_public_key(pem_key.encode())
        return public_key

    @staticmethod
    def public_key_to_pem(public_key: RSAPublicKey):
        """
        Konvertiert ein PublicKey-Objekt in einen PEM-kodierten String.

        Args:
            public_key (PublicKey): Das PublicKey-Objekt.

        Returns:
            str: Der PEM-kodierte öffentliche Schlüssel.
        """
        pem = public_key.public_bytes(
            encoding=serialization.Encoding.PEM,
            format=serialization.PublicFormat.SubjectPublicKeyInfo
        )
        return pem.decode()

decrypt_asymmetric(encrypted_text_hex, private_key_str) staticmethod

Entschlüsselt einen Text mit einem gegebenen privaten Schlüssel.

Parameters:

Name	Type	Description	Default
encrypted_text_hex	str	Der verschlüsselte Text als Hex-String.	required
private_key_str	str	Der private Schlüssel als String.	required

Returns:

Name	Type	Description
str	str	Der entschlüsselte Text.

Source code in toolboxv2/utils/security/cryp.py

@staticmethod
def decrypt_asymmetric(encrypted_text_hex: str, private_key_str: str) -> str:
    """
    Entschlüsselt einen Text mit einem gegebenen privaten Schlüssel.

    Args:
        encrypted_text_hex (str): Der verschlüsselte Text als Hex-String.
        private_key_str (str): Der private Schlüssel als String.

    Returns:
        str: Der entschlüsselte Text.
    """
    try:
        private_key = serialization.load_pem_private_key(private_key_str.encode(), password=None)
        decrypted = private_key.decrypt(
            bytes.fromhex(encrypted_text_hex),
            padding.OAEP(
                mgf=padding.MGF1(algorithm=hashes.SHA512()),
                algorithm=hashes.SHA512(),
                label=None
            )
        )
        return decrypted.decode()

    except Exception as e:
        get_logger().error(f"Error decrypt_asymmetric {e}")
    return "Invalid"

decrypt_symmetric(encrypted_text, key, to_str=True, mute=False) staticmethod

Entschlüsselt einen Text mit einem gegebenen symmetrischen Schlüssel.

Parameters:

Name	Type	Description	Default
encrypted_text	str	Der zu entschlüsselnde Text.	required
key	str	Der symmetrische Schlüssel.	required
to_str	bool	default true returns str if false returns bytes	True

Returns: str: Der entschlüsselte Text.

Source code in toolboxv2/utils/security/cryp.py

@staticmethod
def decrypt_symmetric(encrypted_text: str, key: str, to_str=True, mute=False) -> str or bytes:
    """
    Entschlüsselt einen Text mit einem gegebenen symmetrischen Schlüssel.

    Args:
        encrypted_text (str): Der zu entschlüsselnde Text.
        key (str): Der symmetrische Schlüssel.
        to_str (bool): default true returns str if false returns bytes
    Returns:
        str: Der entschlüsselte Text.
    """

    if isinstance(key, str):
        key = key.encode()

    #try:
    fernet = Fernet(key)
    text_b = fernet.decrypt(encrypted_text)
    if not to_str:
        return text_b
    return text_b.decode()

encrypt_asymmetric(text, public_key_str) staticmethod

Verschlüsselt einen Text mit einem gegebenen öffentlichen Schlüssel.

Parameters:

Name	Type	Description	Default
text	str	Der zu verschlüsselnde Text.	required
public_key_str	str	Der öffentliche Schlüssel als String oder im pem format.	required

Returns:

Name	Type	Description
str	str	Der verschlüsselte Text.

Source code in toolboxv2/utils/security/cryp.py

@staticmethod
def encrypt_asymmetric(text: str, public_key_str: str) -> str:
    """
    Verschlüsselt einen Text mit einem gegebenen öffentlichen Schlüssel.

    Args:
        text (str): Der zu verschlüsselnde Text.
        public_key_str (str): Der öffentliche Schlüssel als String oder im pem format.

    Returns:
        str: Der verschlüsselte Text.
    """
    # try:
    #    public_key: RSAPublicKey = serialization.load_pem_public_key(public_key_str.encode())
    #  except Exception as e:
    #     get_logger().error(f"Error encrypt_asymmetric {e}")
    try:
        public_key: RSAPublicKey = serialization.load_pem_public_key(public_key_str.encode())
        encrypted = public_key.encrypt(
            text.encode(),
            padding.OAEP(
                mgf=padding.MGF1(algorithm=hashes.SHA512()),
                algorithm=hashes.SHA512(),
                label=None
            )
        )
        return encrypted.hex()
    except Exception as e:
        get_logger().error(f"Error encrypt_asymmetric {e}")
        return "Invalid"

encrypt_symmetric(text, key) staticmethod

Verschlüsselt einen Text mit einem gegebenen symmetrischen Schlüssel.

Parameters:

Name	Type	Description	Default
text	str	Der zu verschlüsselnde Text.	required
key	str	Der symmetrische Schlüssel.	required

Returns:

Name	Type	Description
str	str	Der verschlüsselte Text.

Source code in toolboxv2/utils/security/cryp.py

@staticmethod
def encrypt_symmetric(text: str or bytes, key: str) -> str:
    """
    Verschlüsselt einen Text mit einem gegebenen symmetrischen Schlüssel.

    Args:
        text (str): Der zu verschlüsselnde Text.
        key (str): Der symmetrische Schlüssel.

    Returns:
        str: Der verschlüsselte Text.
    """
    if isinstance(text, str):
        text = text.encode()

    try:
        fernet = Fernet(key.encode())
        return fernet.encrypt(text).decode()
    except Exception as e:
        get_logger().error(f"Error encrypt_symmetric #{str(e)}#")
        return "Error encrypt"

generate_asymmetric_keys() staticmethod

Generiert ein Paar von öffentlichen und privaten Schlüsseln für die asymmetrische Verschlüsselung.

Parameters:

Name	Type	Description	Default
seed	int	Ein optionaler Seed-Wert. Standardmäßig None.	required

Returns:

Type	Description
(str, str)	Ein Tupel aus öffentlichem und privatem Schlüssel.

Source code in toolboxv2/utils/security/cryp.py

@staticmethod
def generate_asymmetric_keys() -> (str, str):
    """
    Generiert ein Paar von öffentlichen und privaten Schlüsseln für die asymmetrische Verschlüsselung.

    Args:
        seed (int, optional): Ein optionaler Seed-Wert. Standardmäßig None.

    Returns:
        (str, str): Ein Tupel aus öffentlichem und privatem Schlüssel.
    """
    private_key = rsa.generate_private_key(
        public_exponent=65537,
        key_size=2048 * 3,
    )
    public_key = private_key.public_key()

    # Serialisieren der Schlüssel
    pem_private_key = private_key.private_bytes(
        encoding=serialization.Encoding.PEM,
        format=serialization.PrivateFormat.PKCS8,
        encryption_algorithm=serialization.NoEncryption()
    ).decode()

    pem_public_key = public_key.public_bytes(
        encoding=serialization.Encoding.PEM,
        format=serialization.PublicFormat.SubjectPublicKeyInfo
    ).decode()

    return pem_public_key, pem_private_key

generate_random_string(length) staticmethod

Generiert eine zufällige Zeichenkette der angegebenen Länge.

Parameters:

Name	Type	Description	Default
length	int	Die Länge der zu generierenden Zeichenkette.	required

Returns:

Name	Type	Description
str	str	Die generierte Zeichenkette.

Source code in toolboxv2/utils/security/cryp.py

@staticmethod
def generate_random_string(length: int) -> str:
    """
    Generiert eine zufällige Zeichenkette der angegebenen Länge.

    Args:
        length (int): Die Länge der zu generierenden Zeichenkette.

    Returns:
        str: Die generierte Zeichenkette.
    """
    return secrets.token_urlsafe(length)

generate_seed() staticmethod

Erzeugt eine zufällige Zahl als Seed.

Returns:

Name	Type	Description
int	int	Eine zufällige Zahl.

Source code in toolboxv2/utils/security/cryp.py

@staticmethod
def generate_seed() -> int:
    """
    Erzeugt eine zufällige Zahl als Seed.

    Returns:
        int: Eine zufällige Zahl.
    """
    return random.randint(2 ** 32 - 1, 2 ** 64 - 1)

generate_symmetric_key(as_str=True) staticmethod

Generiert einen Schlüssel für die symmetrische Verschlüsselung.

Returns:

Name	Type	Description
str	str or bytes	Der generierte Schlüssel.

Source code in toolboxv2/utils/security/cryp.py

@staticmethod
def generate_symmetric_key(as_str=True) -> str or bytes:
    """
    Generiert einen Schlüssel für die symmetrische Verschlüsselung.

    Returns:
        str: Der generierte Schlüssel.
    """
    key = Fernet.generate_key()
    if as_str:
        key = key.decode()
    return key

load_keys_from_files(directory='keys') staticmethod

Lädt die Schlüssel aus den Dateien. Der private Schlüssel wird mit dem Device Key entschlüsselt.

Parameters:

Name	Type	Description	Default
directory	str	Das Verzeichnis, aus dem die Schlüssel geladen werden sollen	'keys'

Returns:

Type	Description
(str, str)	Ein Tupel aus öffentlichem und privatem Schlüssel

Raises:

Type	Description
FileNotFoundError	Wenn die Schlüsseldateien nicht gefunden werden können

Source code in toolboxv2/utils/security/cryp.py

@staticmethod
def load_keys_from_files(directory: str = "keys") -> (str, str):
    """
    Lädt die Schlüssel aus den Dateien.
    Der private Schlüssel wird mit dem Device Key entschlüsselt.

    Args:
        directory (str): Das Verzeichnis, aus dem die Schlüssel geladen werden sollen

    Returns:
        (str, str): Ein Tupel aus öffentlichem und privatem Schlüssel

    Raises:
        FileNotFoundError: Wenn die Schlüsseldateien nicht gefunden werden können
    """
    # Pfade zu den Schlüsseldateien
    public_key_path = os.path.join(directory, "public_key.pem")
    private_key_path = os.path.join(directory, "private_key.pem")

    # Prüfe ob die Dateien existieren
    if not os.path.exists(public_key_path) or not os.path.exists(private_key_path):
        return "", ""

    # Hole den Device Key
    device_key = DEVICE_KEY()

    # Lade den öffentlichen Schlüssel
    with open(public_key_path) as f:
        public_key = f.read()

    # Lade und entschlüssele den privaten Schlüssel
    with open(private_key_path) as f:
        encrypted_private_key = f.read()
        private_key = Code.decrypt_symmetric(encrypted_private_key, device_key)

    return public_key, private_key

one_way_hash(text, salt='', pepper='') staticmethod

Erzeugt einen Hash eines gegebenen Textes mit Salt, Pepper und optional einem Seed.

Parameters:

Name	Type	Description	Default
text	str	Der zu hashende Text.	required
salt	str	Der Salt-Wert.	''
pepper	str	Der Pepper-Wert.	''
seed	int	Ein optionaler Seed-Wert. Standardmäßig None.	required

Returns:

Name	Type	Description
str	str	Der resultierende Hash-Wert.

Source code in toolboxv2/utils/security/cryp.py

@staticmethod
def one_way_hash(text: str, salt: str = '', pepper: str = '') -> str:
    """
    Erzeugt einen Hash eines gegebenen Textes mit Salt, Pepper und optional einem Seed.

    Args:
        text (str): Der zu hashende Text.
        salt (str): Der Salt-Wert.
        pepper (str): Der Pepper-Wert.
        seed (int, optional): Ein optionaler Seed-Wert. Standardmäßig None.

    Returns:
        str: Der resultierende Hash-Wert.
    """
    return hashlib.sha256((salt + text + pepper).encode()).hexdigest()

pem_to_public_key(pem_key) staticmethod

Konvertiert einen PEM-kodierten öffentlichen Schlüssel in ein PublicKey-Objekt.

Parameters:

Name	Type	Description	Default
pem_key	str	Der PEM-kodierte öffentliche Schlüssel.	required

Returns:

Name	Type	Description
PublicKey		Das PublicKey-Objekt.

Source code in toolboxv2/utils/security/cryp.py

@staticmethod
def pem_to_public_key(pem_key: str):
    """
    Konvertiert einen PEM-kodierten öffentlichen Schlüssel in ein PublicKey-Objekt.

    Args:
        pem_key (str): Der PEM-kodierte öffentliche Schlüssel.

    Returns:
        PublicKey: Das PublicKey-Objekt.
    """
    public_key = serialization.load_pem_public_key(pem_key.encode())
    return public_key

public_key_to_pem(public_key) staticmethod

Konvertiert ein PublicKey-Objekt in einen PEM-kodierten String.

Parameters:

Name	Type	Description	Default
public_key	PublicKey	Das PublicKey-Objekt.	required

Returns:

Name	Type	Description
str		Der PEM-kodierte öffentliche Schlüssel.

Source code in toolboxv2/utils/security/cryp.py

@staticmethod
def public_key_to_pem(public_key: RSAPublicKey):
    """
    Konvertiert ein PublicKey-Objekt in einen PEM-kodierten String.

    Args:
        public_key (PublicKey): Das PublicKey-Objekt.

    Returns:
        str: Der PEM-kodierte öffentliche Schlüssel.
    """
    pem = public_key.public_bytes(
        encoding=serialization.Encoding.PEM,
        format=serialization.PublicFormat.SubjectPublicKeyInfo
    )
    return pem.decode()

save_keys_to_files(public_key, private_key, directory='keys') staticmethod

Speichert die generierten Schlüssel in separate Dateien. Der private Schlüssel wird mit dem Device Key verschlüsselt.

Parameters:

Name	Type	Description	Default
public_key	str	Der öffentliche Schlüssel im PEM-Format	required
private_key	str	Der private Schlüssel im PEM-Format	required
directory	str	Das Verzeichnis, in dem die Schlüssel gespeichert werden sollen	'keys'

Source code in toolboxv2/utils/security/cryp.py

@staticmethod
def save_keys_to_files(public_key: str, private_key: str, directory: str = "keys") -> None:
    """
    Speichert die generierten Schlüssel in separate Dateien.
    Der private Schlüssel wird mit dem Device Key verschlüsselt.

    Args:
        public_key (str): Der öffentliche Schlüssel im PEM-Format
        private_key (str): Der private Schlüssel im PEM-Format
        directory (str): Das Verzeichnis, in dem die Schlüssel gespeichert werden sollen
    """
    # Erstelle das Verzeichnis, falls es nicht existiert
    os.makedirs(directory, exist_ok=True)

    # Hole den Device Key
    device_key = DEVICE_KEY()

    # Verschlüssele den privaten Schlüssel mit dem Device Key
    encrypted_private_key = Code.encrypt_symmetric(private_key, device_key)

    # Speichere den öffentlichen Schlüssel
    public_key_path = os.path.join(directory, "public_key.pem")
    with open(public_key_path, "w") as f:
        f.write(public_key)

    # Speichere den verschlüsselten privaten Schlüssel
    private_key_path = os.path.join(directory, "private_key.pem")
    with open(private_key_path, "w") as f:
        f.write(encrypted_private_key)

    print("Saved keys in ", public_key_path)

MainTool

Source code in toolboxv2/utils/system/main_tool.py

class MainTool:
    toolID: str = ""
    # app = None
    interface = None
    spec = "app"
    name = ""
    color = "Bold"
    stuf = False

    def __init__(self, *args, **kwargs):
        """
        Standard constructor used for arguments pass
        Do not override. Use __ainit__ instead
        """
        self.__storedargs = args, kwargs
        self.tools = kwargs.get("tool", {})
        self.logger = kwargs.get("logs", get_logger())
        self.color = kwargs.get("color", "WHITE")
        self.todo = kwargs.get("load", kwargs.get("on_start", lambda: None))
        if "on_exit" in kwargs and isinstance(kwargs.get("on_exit"), Callable):
            self.on_exit =self.app.tb(
                mod_name=self.name,
                name=kwargs.get("on_exit").__name__,
                version=self.version if hasattr(self, 'version') else "0.0.0",
            )(kwargs.get("on_exit"))
        self.async_initialized = False
        if self.todo:
            try:
                if inspect.iscoroutinefunction(self.todo):
                    pass
                else:
                    self.todo()
                get_logger().info(f"{self.name} on load suspended")
            except Exception as e:
                get_logger().error(f"Error loading mod {self.name} {e}")
                if self.app.debug:
                    import traceback
                    traceback.print_exc()
        else:
            get_logger().info(f"{self.name} no load require")

    async def __ainit__(self, *args, **kwargs):
        self.version = kwargs.get("v", kwargs.get("version", "0.0.0"))
        self.tools = kwargs.get("tool", {})
        self.name = kwargs["name"]
        self.logger = kwargs.get("logs", get_logger())
        self.color = kwargs.get("color", "WHITE")
        self.todo = kwargs.get("load", kwargs.get("on_start"))
        if not hasattr(self, 'config'):
            self.config = {}
        self.user = None
        self.description = "A toolbox mod" if kwargs.get("description") is None else kwargs.get("description")
        if MainTool.interface is None:
            MainTool.interface = self.app.interface_type
        # Result.default(self.app.interface)

        if self.todo:
            try:
                if inspect.iscoroutinefunction(self.todo):
                    await self.todo()
                else:
                    pass
                await asyncio.sleep(0.1)
                get_logger().info(f"{self.name} on load suspended")
            except Exception as e:
                get_logger().error(f"Error loading mod {self.name} {e}")
                if self.app.debug:
                    import traceback
                    traceback.print_exc()
        else:
            get_logger().info(f"{self.name} no load require")
        self.app.print(f"TOOL : {self.spec}.{self.name} online")



    @property
    def app(self):
        return get_app(
            from_=f"{self.spec}.{self.name}|{self.toolID if self.toolID else '*' + MainTool.toolID} {self.interface if self.interface else MainTool.interface}")

    @app.setter
    def app(self, v):
        raise PermissionError(f"You cannot set the App Instance! {v=}")

    @staticmethod
    def return_result(error: ToolBoxError = ToolBoxError.none,
                      exec_code: int = 0,
                      help_text: str = "",
                      data_info=None,
                      data=None,
                      data_to=None):

        if data_to is None:
            data_to = MainTool.interface if MainTool.interface is not None else ToolBoxInterfaces.cli

        if data is None:
            data = {}

        if data_info is None:
            data_info = {}

        return Result(
            error,
            ToolBoxResult(data_info=data_info, data=data, data_to=data_to),
            ToolBoxInfo(exec_code=exec_code, help_text=help_text)
        )

    def print(self, message, end="\n", **kwargs):
        if self.stuf:
            return

        self.app.print(Style.style_dic[self.color] + self.name + Style.style_dic["END"] + ":", message, end=end,
                       **kwargs)

    def add_str_to_config(self, command):
        if len(command) != 2:
            self.logger.error('Invalid command must be key value')
            return False
        self.config[command[0]] = command[1]

    def webInstall(self, user_instance, construct_render) -> str:
        """"Returns a web installer for the given user instance and construct render template"""

    def get_version(self) -> str:
        """"Returns the version"""
        return self.version

    async def get_user(self, username: str) -> Result:
        return await self.app.a_run_any(CLOUDM_AUTHMANAGER.GET_USER_BY_NAME, username=username, get_results=True)

    async def __initobj(self):
        """Crutch used for __await__ after spawning"""
        assert not self.async_initialized
        self.async_initialized = True
        # pass the parameters to __ainit__ that passed to __init__
        await self.__ainit__(*self.__storedargs[0], **self.__storedargs[1])
        return self

    def __await__(self):
        return self.__initobj().__await__()

__init__(*args, **kwargs)

Standard constructor used for arguments pass Do not override. Use ainit instead

Source code in toolboxv2/utils/system/main_tool.py

def __init__(self, *args, **kwargs):
    """
    Standard constructor used for arguments pass
    Do not override. Use __ainit__ instead
    """
    self.__storedargs = args, kwargs
    self.tools = kwargs.get("tool", {})
    self.logger = kwargs.get("logs", get_logger())
    self.color = kwargs.get("color", "WHITE")
    self.todo = kwargs.get("load", kwargs.get("on_start", lambda: None))
    if "on_exit" in kwargs and isinstance(kwargs.get("on_exit"), Callable):
        self.on_exit =self.app.tb(
            mod_name=self.name,
            name=kwargs.get("on_exit").__name__,
            version=self.version if hasattr(self, 'version') else "0.0.0",
        )(kwargs.get("on_exit"))
    self.async_initialized = False
    if self.todo:
        try:
            if inspect.iscoroutinefunction(self.todo):
                pass
            else:
                self.todo()
            get_logger().info(f"{self.name} on load suspended")
        except Exception as e:
            get_logger().error(f"Error loading mod {self.name} {e}")
            if self.app.debug:
                import traceback
                traceback.print_exc()
    else:
        get_logger().info(f"{self.name} no load require")

__initobj() async

Crutch used for await after spawning

Source code in toolboxv2/utils/system/main_tool.py

async def __initobj(self):
    """Crutch used for __await__ after spawning"""
    assert not self.async_initialized
    self.async_initialized = True
    # pass the parameters to __ainit__ that passed to __init__
    await self.__ainit__(*self.__storedargs[0], **self.__storedargs[1])
    return self

get_version()

"Returns the version

Source code in toolboxv2/utils/system/main_tool.py

def get_version(self) -> str:
    """"Returns the version"""
    return self.version

webInstall(user_instance, construct_render)

"Returns a web installer for the given user instance and construct render template

Source code in toolboxv2/utils/system/main_tool.py

def webInstall(self, user_instance, construct_render) -> str:
    """"Returns a web installer for the given user instance and construct render template"""

Result

Source code in toolboxv2/utils/system/types.py

class Result(Generic[T]):
    _task = None
    _generic_type: Optional[Type] = None

    def __init__(self,
                 error: ToolBoxError,
                 result: ToolBoxResult,
                 info: ToolBoxInfo,
                 origin: Any | None = None,
                 generic_type: Optional[Type] = None
                 ):
        self.error: ToolBoxError = error
        self.result: ToolBoxResult = result
        self.info: ToolBoxInfo = info
        self.origin = origin
        self._generic_type = generic_type

    def __class_getitem__(cls, item):
        """Enable Result[Type] syntax"""

        class TypedResult(cls):
            _generic_type = item

            def __init__(self, *args, **kwargs):
                super().__init__(*args, **kwargs)
                self._generic_type = item

        return TypedResult

    def typed_get(self, key=None, default=None) -> T:
        """Get data with type validation"""
        data = self.get(key, default)

        if self._generic_type and data is not None:
            # Validate type matches generic parameter
            if not self._validate_type(data, self._generic_type):
                from toolboxv2 import get_logger
                get_logger().warning(f"Type mismatch: expected {self._generic_type}, got {type(data)}")

        return data

    async def typed_aget(self, key=None, default=None) -> T:
        """Async get data with type validation"""
        data = await self.aget(key, default)

        if self._generic_type and data is not None:
            if not self._validate_type(data, self._generic_type):
                from toolboxv2 import get_logger
                get_logger().warning(f"Type mismatch: expected {self._generic_type}, got {type(data)}")

        return data

    def _validate_type(self, data, expected_type) -> bool:
        """Validate data matches expected type"""
        try:
            # Handle List[Type] syntax
            origin = get_origin(expected_type)
            if origin is list or origin is List:
                if not isinstance(data, list):
                    return False

                # Check list element types if specified
                args = get_args(expected_type)
                if args and data:
                    element_type = args[0]
                    return all(isinstance(item, element_type) for item in data)
                return True

            # Handle other generic types
            elif origin is not None:
                return isinstance(data, origin)

            # Handle regular types
            else:
                return isinstance(data, expected_type)

        except Exception:
            return True  # Skip validation on error

    @classmethod
    def typed_ok(cls, data: T, data_info="", info="OK", interface=ToolBoxInterfaces.native) -> 'Result[T]':
        """Create OK result with type information"""
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=0, help_text=info)
        result = ToolBoxResult(data_to=interface, data=data, data_info=data_info, data_type=type(data).__name__)

        instance = cls(error=error, info=info_obj, result=result)
        if hasattr(cls, '_generic_type'):
            instance._generic_type = cls._generic_type

        return instance

    @classmethod
    def typed_json(cls, data: T, info="OK", interface=ToolBoxInterfaces.remote, exec_code=0,
                   status_code=None) -> 'Result[T]':
        """Create JSON result with type information"""
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=status_code or exec_code, help_text=info)

        result = ToolBoxResult(
            data_to=interface,
            data=data,
            data_info="JSON response",
            data_type="json"
        )

        instance = cls(error=error, info=info_obj, result=result)
        if hasattr(cls, '_generic_type'):
            instance._generic_type = cls._generic_type

        return instance

    def cast_to(self, target_type: Type[T]) -> 'Result[T]':
        """Cast result to different type"""
        new_result = Result(
            error=self.error,
            result=self.result,
            info=self.info,
            origin=self.origin,
            generic_type=target_type
        )
        new_result._generic_type = target_type
        return new_result

    def get_type_info(self) -> Optional[Type]:
        """Get the generic type information"""
        return self._generic_type

    def is_typed(self) -> bool:
        """Check if result has type information"""
        return self._generic_type is not None

    def as_result(self):
        return self

    def as_dict(self):
        return {
            "error":self.error.value if isinstance(self.error, Enum) else self.error,
        "result" : {
            "data_to":self.result.data_to.value if isinstance(self.result.data_to, Enum) else self.result.data_to,
            "data_info":self.result.data_info,
            "data":self.result.data,
            "data_type":self.result.data_type
        } if self.result else None,
        "info" : {
            "exec_code" : self.info.exec_code,  # exec_code umwandel in http resposn codes
        "help_text" : self.info.help_text
        } if self.info else None,
        "origin" : self.origin
        }

    def set_origin(self, origin):
        if self.origin is not None:
            raise ValueError("You cannot Change the origin of a Result!")
        self.origin = origin
        return self

    def set_dir_origin(self, name, extras="assets/"):
        if self.origin is not None:
            raise ValueError("You cannot Change the origin of a Result!")
        self.origin = f"mods/{name}/{extras}"
        return self

    def is_error(self):
        if _test_is_result(self.result.data):
            return self.result.data.is_error()
        if self.error == ToolBoxError.none:
            return False
        if self.info.exec_code == 0:
            return False
        return self.info.exec_code != 200

    def is_ok(self):
        return not self.is_error()

    def is_data(self):
        return self.result.data is not None

    def to_api_result(self):
        # print(f" error={self.error}, result= {self.result}, info= {self.info}, origin= {self.origin}")
        return ApiResult(
            error=self.error.value if isinstance(self.error, Enum) else self.error,
            result=ToolBoxResultBM(
                data_to=self.result.data_to.value if isinstance(self.result.data_to, Enum) else self.result.data_to,
                data_info=self.result.data_info,
                data=self.result.data,
                data_type=self.result.data_type
            ) if self.result else None,
            info=ToolBoxInfoBM(
                exec_code=self.info.exec_code,  # exec_code umwandel in http resposn codes
                help_text=self.info.help_text
            ) if self.info else None,
            origin=self.origin
        )

    def task(self, task):
        self._task = task
        return self

    @staticmethod
    def result_from_dict(error: str, result: dict, info: dict, origin: list or None or str):
        # print(f" error={self.error}, result= {self.result}, info= {self.info}, origin= {self.origin}")
        return ApiResult(
            error=error if isinstance(error, Enum) else error,
            result=ToolBoxResultBM(
                data_to=result.get('data_to') if isinstance(result.get('data_to'), Enum) else result.get('data_to'),
                data_info=result.get('data_info', '404'),
                data=result.get('data'),
                data_type=result.get('data_type', '404'),
            ) if result else ToolBoxResultBM(
                data_to=ToolBoxInterfaces.cli.value,
                data_info='',
                data='404',
                data_type='404',
            ),
            info=ToolBoxInfoBM(
                exec_code=info.get('exec_code', 404),
                help_text=info.get('help_text', '404')
            ) if info else ToolBoxInfoBM(
                exec_code=404,
                help_text='404'
            ),
            origin=origin
        ).as_result()

    @classmethod
    def stream(cls,
               stream_generator: Any,  # Renamed from source for clarity
               content_type: str = "text/event-stream",  # Default to SSE
               headers: dict | None = None,
               info: str = "OK",
               interface: ToolBoxInterfaces = ToolBoxInterfaces.remote,
               cleanup_func: Callable[[], None] | Callable[[], T] | Callable[[], AsyncGenerator[T, None]] | None = None):
        """
        Create a streaming response Result. Handles SSE and other stream types.

        Args:
            stream_generator: Any stream source (async generator, sync generator, iterable, or single item).
            content_type: Content-Type header (default: text/event-stream for SSE).
            headers: Additional HTTP headers for the response.
            info: Help text for the result.
            interface: Interface to send data to.
            cleanup_func: Optional function for cleanup.

        Returns:
            A Result object configured for streaming.
        """
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=0, help_text=info)

        final_generator: AsyncGenerator[str, None]

        if content_type == "text/event-stream":
            # For SSE, always use SSEGenerator.create_sse_stream to wrap the source.
            # SSEGenerator.create_sse_stream handles various types of stream_generator internally.
            final_generator = SSEGenerator.create_sse_stream(source=stream_generator, cleanup_func=cleanup_func)

            # Standard SSE headers for the HTTP response itself
            # These will be stored in the Result object. Rust side decides how to use them.
            standard_sse_headers = {
                "Cache-Control": "no-cache",  # SSE specific
                "Connection": "keep-alive",  # SSE specific
                "X-Accel-Buffering": "no",  # Useful for proxies with SSE
                # Content-Type is implicitly text/event-stream, will be in streaming_data below
            }
            all_response_headers = standard_sse_headers.copy()
            if headers:
                all_response_headers.update(headers)
        else:
            # For non-SSE streams.
            # If stream_generator is sync, wrap it to be async.
            # If already async or single item, it will be handled.
            # Rust's stream_generator in ToolboxClient seems to handle both sync/async Python generators.
            # For consistency with how SSEGenerator does it, we can wrap sync ones.
            if inspect.isgenerator(stream_generator) or \
                (not isinstance(stream_generator, str) and hasattr(stream_generator, '__iter__')):
                final_generator = SSEGenerator.wrap_sync_generator(stream_generator)  # Simple async wrapper
            elif inspect.isasyncgen(stream_generator):
                final_generator = stream_generator
            else:  # Single item or string
                async def _single_item_gen():
                    yield stream_generator

                final_generator = _single_item_gen()
            all_response_headers = headers if headers else {}

        # Prepare streaming data to be stored in the Result object
        streaming_data = {
            "type": "stream",  # Indicator for Rust side
            "generator": final_generator,
            "content_type": content_type,  # Let Rust know the intended content type
            "headers": all_response_headers  # Intended HTTP headers for the overall response
        }

        result_payload = ToolBoxResult(
            data_to=interface,
            data=streaming_data,
            data_info="Streaming response" if content_type != "text/event-stream" else "SSE Event Stream",
            data_type="stream"  # Generic type for Rust to identify it needs to stream from 'generator'
        )

        return cls(error=error, info=info_obj, result=result_payload)

    @classmethod
    def sse(cls,
            stream_generator: Any,
            info: str = "OK",
            interface: ToolBoxInterfaces = ToolBoxInterfaces.remote,
            cleanup_func: Callable[[], None] | Callable[[], T] | Callable[[], AsyncGenerator[T, None]] | None = None,
            # http_headers: Optional[dict] = None # If we want to allow overriding default SSE HTTP headers
            ):
        """
        Create an Server-Sent Events (SSE) streaming response Result.

        Args:
            stream_generator: A source yielding individual data items. This can be an
                              async generator, sync generator, iterable, or a single item.
                              Each item will be formatted as an SSE event.
            info: Optional help text for the Result.
            interface: Optional ToolBoxInterface to target.
            cleanup_func: Optional cleanup function to run when the stream ends or is cancelled.
            #http_headers: Optional dictionary of custom HTTP headers for the SSE response.

        Returns:
            A Result object configured for SSE streaming.
        """
        # Result.stream will handle calling SSEGenerator.create_sse_stream
        # and setting appropriate default headers for SSE when content_type is "text/event-stream".
        return cls.stream(
            stream_generator=stream_generator,
            content_type="text/event-stream",
            # headers=http_headers, # Pass if we add http_headers param
            info=info,
            interface=interface,
            cleanup_func=cleanup_func
        )

    @classmethod
    def default(cls, interface=ToolBoxInterfaces.native):
        error = ToolBoxError.none
        info = ToolBoxInfo(exec_code=-1, help_text="")
        result = ToolBoxResult(data_to=interface)
        return cls(error=error, info=info, result=result)

    @classmethod
    def json(cls, data, info="OK", interface=ToolBoxInterfaces.remote, exec_code=0, status_code=None):
        """Create a JSON response Result."""
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=status_code or exec_code, help_text=info)

        result = ToolBoxResult(
            data_to=interface,
            data=data,
            data_info="JSON response",
            data_type="json"
        )

        return cls(error=error, info=info_obj, result=result)

    @classmethod
    def text(cls, text_data, content_type="text/plain",exec_code=None,status=200, info="OK", interface=ToolBoxInterfaces.remote, headers=None):
        """Create a text response Result with specific content type."""
        if headers is not None:
            return cls.html(text_data, status= exec_code or status, info=info, headers=headers)
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=exec_code or status, help_text=info)

        result = ToolBoxResult(
            data_to=interface,
            data=text_data,
            data_info="Text response",
            data_type=content_type
        )

        return cls(error=error, info=info_obj, result=result)

    @classmethod
    def binary(cls, data, content_type="application/octet-stream", download_name=None, info="OK",
               interface=ToolBoxInterfaces.remote):
        """Create a binary data response Result."""
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=0, help_text=info)

        # Create a dictionary with binary data and metadata
        binary_data = {
            "data": data,
            "content_type": content_type,
            "filename": download_name
        }

        result = ToolBoxResult(
            data_to=interface,
            data=binary_data,
            data_info=f"Binary response: {download_name}" if download_name else "Binary response",
            data_type="binary"
        )

        return cls(error=error, info=info_obj, result=result)

    @classmethod
    def file(cls, data, filename, content_type=None, info="OK", interface=ToolBoxInterfaces.remote):
        """Create a file download response Result.

        Args:
            data: File data as bytes or base64 string
            filename: Name of the file for download
            content_type: MIME type of the file (auto-detected if None)
            info: Response info text
            interface: Target interface

        Returns:
            Result object configured for file download
        """
        import base64
        import mimetypes

        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=200, help_text=info)

        # Auto-detect content type if not provided
        if content_type is None:
            content_type, _ = mimetypes.guess_type(filename)
            if content_type is None:
                content_type = "application/octet-stream"

        # Ensure data is base64 encoded string (as expected by Rust server)
        if isinstance(data, bytes):
            base64_data = base64.b64encode(data).decode('utf-8')
        elif isinstance(data, str):
            # Assume it's already base64 encoded
            base64_data = data
        else:
            raise ValueError("File data must be bytes or base64 string")

        result = ToolBoxResult(
            data_to=interface,
            data=base64_data,  # Rust expects base64 string for "file" type
            data_info=f"File download: {filename}",
            data_type="file"
        )

        return cls(error=error, info=info_obj, result=result)

    @classmethod
    def redirect(cls, url, status_code=302, info="Redirect", interface=ToolBoxInterfaces.remote):
        """Create a redirect response."""
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=status_code, help_text=info)

        result = ToolBoxResult(
            data_to=interface,
            data=url,
            data_info="Redirect response",
            data_type="redirect"
        )

        return cls(error=error, info=info_obj, result=result)

    @classmethod
    def ok(cls, data=None, data_info="", info="OK", interface=ToolBoxInterfaces.native):
        error = ToolBoxError.none
        info = ToolBoxInfo(exec_code=0, help_text=info)
        result = ToolBoxResult(data_to=interface, data=data, data_info=data_info, data_type=type(data).__name__)
        return cls(error=error, info=info, result=result)

    @classmethod
    def html(cls, data=None, data_info="", info="OK", interface=ToolBoxInterfaces.remote, data_type="html",status=200, headers=None, row=False):
        error = ToolBoxError.none
        info = ToolBoxInfo(exec_code=status, help_text=info)
        from ...utils.system.getting_and_closing_app import get_app

        if not row and not '"<div class="main-content""' in data:
            data = f'<div class="main-content frosted-glass">{data}<div>'
        if not row and not get_app().web_context() in data:
            data = get_app().web_context() + data

        if isinstance(headers, dict):
            result = ToolBoxResult(data_to=interface, data={'html':data,'headers':headers}, data_info=data_info,
                                   data_type="special_html")
        else:
            result = ToolBoxResult(data_to=interface, data=data, data_info=data_info,
                                   data_type=data_type if data_type is not None else type(data).__name__)
        return cls(error=error, info=info, result=result)

    @classmethod
    def future(cls, data=None, data_info="", info="OK", interface=ToolBoxInterfaces.future):
        error = ToolBoxError.none
        info = ToolBoxInfo(exec_code=0, help_text=info)
        result = ToolBoxResult(data_to=interface, data=data, data_info=data_info, data_type="future")
        return cls(error=error, info=info, result=result)

    @classmethod
    def custom_error(cls, data=None, data_info="", info="", exec_code=-1, interface=ToolBoxInterfaces.native):
        error = ToolBoxError.custom_error
        info = ToolBoxInfo(exec_code=exec_code, help_text=info)
        result = ToolBoxResult(data_to=interface, data=data, data_info=data_info, data_type=type(data).__name__)
        return cls(error=error, info=info, result=result)

    @classmethod
    def error(cls, data=None, data_info="", info="", exec_code=450, interface=ToolBoxInterfaces.remote):
        error = ToolBoxError.custom_error
        info = ToolBoxInfo(exec_code=exec_code, help_text=info)
        result = ToolBoxResult(data_to=interface, data=data, data_info=data_info, data_type=type(data).__name__)
        return cls(error=error, info=info, result=result)

    @classmethod
    def default_user_error(cls, info="", exec_code=-3, interface=ToolBoxInterfaces.native, data=None):
        error = ToolBoxError.input_error
        info = ToolBoxInfo(exec_code, info)
        result = ToolBoxResult(data_to=interface, data=data, data_type=type(data).__name__)
        return cls(error=error, info=info, result=result)

    @classmethod
    def default_internal_error(cls, info="", exec_code=-2, interface=ToolBoxInterfaces.native, data=None):
        error = ToolBoxError.internal_error
        info = ToolBoxInfo(exec_code, info)
        result = ToolBoxResult(data_to=interface, data=data, data_type=type(data).__name__)
        return cls(error=error, info=info, result=result)

    def print(self, show=True, show_data=True, prifix="", full_data=False):
        data = '\n' + f"{((prifix + f'Data_{self.result.data_type}: ' + str(self.result.data) if self.result.data is not None else 'NO Data') if not isinstance(self.result.data, Result) else self.result.data.print(show=False, show_data=show_data, prifix=prifix + '-')) if show_data else 'Data: private'}"
        origin = '\n' + f"{prifix + 'Origin: ' + str(self.origin) if self.origin is not None else 'NO Origin'}"
        text = (f"Function Exec code: {self.info.exec_code}"
                f"\n{prifix}Info's:"
                f" {self.info.help_text} {'<|> ' + str(self.result.data_info) if self.result.data_info is not None else ''}"
                f"{origin}{((data[:100]+'...') if not full_data else (data)) if not data.endswith('NO Data') else ''}\n")
        if not show:
            return text
        print("\n======== Result ========\n" + text + "------- EndOfD -------")
        return self

    def log(self, show_data=True, prifix=""):
        from toolboxv2 import get_logger
        get_logger().debug(self.print(show=False, show_data=show_data, prifix=prifix).replace("\n", " - "))
        return self

    def __str__(self):
        return self.print(show=False, show_data=True)

    def get(self, key=None, default=None):
        data = self.result.data
        if isinstance(data, Result):
            return data.get(key=key, default=default)
        if key is not None and isinstance(data, dict):
            return data.get(key, default)
        return data if data is not None else default

    async def aget(self, key=None, default=None):
        if asyncio.isfuture(self.result.data) or asyncio.iscoroutine(self.result.data) or (
            isinstance(self.result.data_to, Enum) and self.result.data_to.name == ToolBoxInterfaces.future.name):
            data = await self.result.data
        else:
            data = self.get(key=None, default=None)
        if isinstance(data, Result):
            return data.get(key=key, default=default)
        if key is not None and isinstance(data, dict):
            return data.get(key, default)
        return data if data is not None else default

    def lazy_return(self, _=0, data=None, **kwargs):
        flags = ['raise', 'logg', 'user', 'intern']
        flag = flags[_] if isinstance(_, int) else _
        if self.info.exec_code == 0:
            return self if data is None else data if _test_is_result(data) else self.ok(data=data, **kwargs)
        if flag == 'raise':
            raise ValueError(self.print(show=False))
        if flag == 'logg':
            from .. import get_logger
            get_logger().error(self.print(show=False))

        if flag == 'user':
            return self if data is None else data if _test_is_result(data) else self.default_user_error(data=data,
                                                                                                        **kwargs)
        if flag == 'intern':
            return self if data is None else data if _test_is_result(data) else self.default_internal_error(data=data,
                                                                                                            **kwargs)

        return self if data is None else data if _test_is_result(data) else self.custom_error(data=data, **kwargs)

    @property
    def bg_task(self):
        return self._task

__class_getitem__(item)

Enable Result[Type] syntax

Source code in toolboxv2/utils/system/types.py

def __class_getitem__(cls, item):
    """Enable Result[Type] syntax"""

    class TypedResult(cls):
        _generic_type = item

        def __init__(self, *args, **kwargs):
            super().__init__(*args, **kwargs)
            self._generic_type = item

    return TypedResult

binary(data, content_type='application/octet-stream', download_name=None, info='OK', interface=ToolBoxInterfaces.remote) classmethod

Create a binary data response Result.

Source code in toolboxv2/utils/system/types.py

@classmethod
def binary(cls, data, content_type="application/octet-stream", download_name=None, info="OK",
           interface=ToolBoxInterfaces.remote):
    """Create a binary data response Result."""
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=0, help_text=info)

    # Create a dictionary with binary data and metadata
    binary_data = {
        "data": data,
        "content_type": content_type,
        "filename": download_name
    }

    result = ToolBoxResult(
        data_to=interface,
        data=binary_data,
        data_info=f"Binary response: {download_name}" if download_name else "Binary response",
        data_type="binary"
    )

    return cls(error=error, info=info_obj, result=result)

cast_to(target_type)

Cast result to different type

Source code in toolboxv2/utils/system/types.py

def cast_to(self, target_type: Type[T]) -> 'Result[T]':
    """Cast result to different type"""
    new_result = Result(
        error=self.error,
        result=self.result,
        info=self.info,
        origin=self.origin,
        generic_type=target_type
    )
    new_result._generic_type = target_type
    return new_result

file(data, filename, content_type=None, info='OK', interface=ToolBoxInterfaces.remote) classmethod

Create a file download response Result.

Parameters:

Name	Type	Description	Default
data		File data as bytes or base64 string	required
filename		Name of the file for download	required
content_type		MIME type of the file (auto-detected if None)	None
info		Response info text	'OK'
interface		Target interface	remote

Returns:

Type	Description
	Result object configured for file download

Source code in toolboxv2/utils/system/types.py

@classmethod
def file(cls, data, filename, content_type=None, info="OK", interface=ToolBoxInterfaces.remote):
    """Create a file download response Result.

    Args:
        data: File data as bytes or base64 string
        filename: Name of the file for download
        content_type: MIME type of the file (auto-detected if None)
        info: Response info text
        interface: Target interface

    Returns:
        Result object configured for file download
    """
    import base64
    import mimetypes

    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=200, help_text=info)

    # Auto-detect content type if not provided
    if content_type is None:
        content_type, _ = mimetypes.guess_type(filename)
        if content_type is None:
            content_type = "application/octet-stream"

    # Ensure data is base64 encoded string (as expected by Rust server)
    if isinstance(data, bytes):
        base64_data = base64.b64encode(data).decode('utf-8')
    elif isinstance(data, str):
        # Assume it's already base64 encoded
        base64_data = data
    else:
        raise ValueError("File data must be bytes or base64 string")

    result = ToolBoxResult(
        data_to=interface,
        data=base64_data,  # Rust expects base64 string for "file" type
        data_info=f"File download: {filename}",
        data_type="file"
    )

    return cls(error=error, info=info_obj, result=result)

get_type_info()

Get the generic type information

Source code in toolboxv2/utils/system/types.py

def get_type_info(self) -> Optional[Type]:
    """Get the generic type information"""
    return self._generic_type

is_typed()

Check if result has type information

Source code in toolboxv2/utils/system/types.py

def is_typed(self) -> bool:
    """Check if result has type information"""
    return self._generic_type is not None

json(data, info='OK', interface=ToolBoxInterfaces.remote, exec_code=0, status_code=None) classmethod

Create a JSON response Result.

Source code in toolboxv2/utils/system/types.py

@classmethod
def json(cls, data, info="OK", interface=ToolBoxInterfaces.remote, exec_code=0, status_code=None):
    """Create a JSON response Result."""
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=status_code or exec_code, help_text=info)

    result = ToolBoxResult(
        data_to=interface,
        data=data,
        data_info="JSON response",
        data_type="json"
    )

    return cls(error=error, info=info_obj, result=result)

redirect(url, status_code=302, info='Redirect', interface=ToolBoxInterfaces.remote) classmethod

Create a redirect response.

Source code in toolboxv2/utils/system/types.py

@classmethod
def redirect(cls, url, status_code=302, info="Redirect", interface=ToolBoxInterfaces.remote):
    """Create a redirect response."""
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=status_code, help_text=info)

    result = ToolBoxResult(
        data_to=interface,
        data=url,
        data_info="Redirect response",
        data_type="redirect"
    )

    return cls(error=error, info=info_obj, result=result)

sse(stream_generator, info='OK', interface=ToolBoxInterfaces.remote, cleanup_func=None) classmethod

Create an Server-Sent Events (SSE) streaming response Result.

Parameters:

Name	Type	Description	Default
stream_generator	Any	A source yielding individual data items. This can be an async generator, sync generator, iterable, or a single item. Each item will be formatted as an SSE event.	required
info	str	Optional help text for the Result.	'OK'
interface	ToolBoxInterfaces	Optional ToolBoxInterface to target.	remote
cleanup_func	Callable[[], None] | Callable[[], T] | Callable[[], AsyncGenerator[T, None]] | None	Optional cleanup function to run when the stream ends or is cancelled.	None
#http_headers		Optional dictionary of custom HTTP headers for the SSE response.	required

Returns:

Type	Description
	A Result object configured for SSE streaming.

Source code in toolboxv2/utils/system/types.py

@classmethod
def sse(cls,
        stream_generator: Any,
        info: str = "OK",
        interface: ToolBoxInterfaces = ToolBoxInterfaces.remote,
        cleanup_func: Callable[[], None] | Callable[[], T] | Callable[[], AsyncGenerator[T, None]] | None = None,
        # http_headers: Optional[dict] = None # If we want to allow overriding default SSE HTTP headers
        ):
    """
    Create an Server-Sent Events (SSE) streaming response Result.

    Args:
        stream_generator: A source yielding individual data items. This can be an
                          async generator, sync generator, iterable, or a single item.
                          Each item will be formatted as an SSE event.
        info: Optional help text for the Result.
        interface: Optional ToolBoxInterface to target.
        cleanup_func: Optional cleanup function to run when the stream ends or is cancelled.
        #http_headers: Optional dictionary of custom HTTP headers for the SSE response.

    Returns:
        A Result object configured for SSE streaming.
    """
    # Result.stream will handle calling SSEGenerator.create_sse_stream
    # and setting appropriate default headers for SSE when content_type is "text/event-stream".
    return cls.stream(
        stream_generator=stream_generator,
        content_type="text/event-stream",
        # headers=http_headers, # Pass if we add http_headers param
        info=info,
        interface=interface,
        cleanup_func=cleanup_func
    )

stream(stream_generator, content_type='text/event-stream', headers=None, info='OK', interface=ToolBoxInterfaces.remote, cleanup_func=None) classmethod

Create a streaming response Result. Handles SSE and other stream types.

Parameters:

Name	Type	Description	Default
stream_generator	Any	Any stream source (async generator, sync generator, iterable, or single item).	required
content_type	str	Content-Type header (default: text/event-stream for SSE).	'text/event-stream'
headers	dict | None	Additional HTTP headers for the response.	None
info	str	Help text for the result.	'OK'
interface	ToolBoxInterfaces	Interface to send data to.	remote
cleanup_func	Callable[[], None] | Callable[[], T] | Callable[[], AsyncGenerator[T, None]] | None	Optional function for cleanup.	None

Returns:

Type	Description
	A Result object configured for streaming.

Source code in toolboxv2/utils/system/types.py

@classmethod
def stream(cls,
           stream_generator: Any,  # Renamed from source for clarity
           content_type: str = "text/event-stream",  # Default to SSE
           headers: dict | None = None,
           info: str = "OK",
           interface: ToolBoxInterfaces = ToolBoxInterfaces.remote,
           cleanup_func: Callable[[], None] | Callable[[], T] | Callable[[], AsyncGenerator[T, None]] | None = None):
    """
    Create a streaming response Result. Handles SSE and other stream types.

    Args:
        stream_generator: Any stream source (async generator, sync generator, iterable, or single item).
        content_type: Content-Type header (default: text/event-stream for SSE).
        headers: Additional HTTP headers for the response.
        info: Help text for the result.
        interface: Interface to send data to.
        cleanup_func: Optional function for cleanup.

    Returns:
        A Result object configured for streaming.
    """
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=0, help_text=info)

    final_generator: AsyncGenerator[str, None]

    if content_type == "text/event-stream":
        # For SSE, always use SSEGenerator.create_sse_stream to wrap the source.
        # SSEGenerator.create_sse_stream handles various types of stream_generator internally.
        final_generator = SSEGenerator.create_sse_stream(source=stream_generator, cleanup_func=cleanup_func)

        # Standard SSE headers for the HTTP response itself
        # These will be stored in the Result object. Rust side decides how to use them.
        standard_sse_headers = {
            "Cache-Control": "no-cache",  # SSE specific
            "Connection": "keep-alive",  # SSE specific
            "X-Accel-Buffering": "no",  # Useful for proxies with SSE
            # Content-Type is implicitly text/event-stream, will be in streaming_data below
        }
        all_response_headers = standard_sse_headers.copy()
        if headers:
            all_response_headers.update(headers)
    else:
        # For non-SSE streams.
        # If stream_generator is sync, wrap it to be async.
        # If already async or single item, it will be handled.
        # Rust's stream_generator in ToolboxClient seems to handle both sync/async Python generators.
        # For consistency with how SSEGenerator does it, we can wrap sync ones.
        if inspect.isgenerator(stream_generator) or \
            (not isinstance(stream_generator, str) and hasattr(stream_generator, '__iter__')):
            final_generator = SSEGenerator.wrap_sync_generator(stream_generator)  # Simple async wrapper
        elif inspect.isasyncgen(stream_generator):
            final_generator = stream_generator
        else:  # Single item or string
            async def _single_item_gen():
                yield stream_generator

            final_generator = _single_item_gen()
        all_response_headers = headers if headers else {}

    # Prepare streaming data to be stored in the Result object
    streaming_data = {
        "type": "stream",  # Indicator for Rust side
        "generator": final_generator,
        "content_type": content_type,  # Let Rust know the intended content type
        "headers": all_response_headers  # Intended HTTP headers for the overall response
    }

    result_payload = ToolBoxResult(
        data_to=interface,
        data=streaming_data,
        data_info="Streaming response" if content_type != "text/event-stream" else "SSE Event Stream",
        data_type="stream"  # Generic type for Rust to identify it needs to stream from 'generator'
    )

    return cls(error=error, info=info_obj, result=result_payload)

text(text_data, content_type='text/plain', exec_code=None, status=200, info='OK', interface=ToolBoxInterfaces.remote, headers=None) classmethod

Create a text response Result with specific content type.

Source code in toolboxv2/utils/system/types.py

@classmethod
def text(cls, text_data, content_type="text/plain",exec_code=None,status=200, info="OK", interface=ToolBoxInterfaces.remote, headers=None):
    """Create a text response Result with specific content type."""
    if headers is not None:
        return cls.html(text_data, status= exec_code or status, info=info, headers=headers)
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=exec_code or status, help_text=info)

    result = ToolBoxResult(
        data_to=interface,
        data=text_data,
        data_info="Text response",
        data_type=content_type
    )

    return cls(error=error, info=info_obj, result=result)

typed_aget(key=None, default=None) async

Async get data with type validation

Source code in toolboxv2/utils/system/types.py

async def typed_aget(self, key=None, default=None) -> T:
    """Async get data with type validation"""
    data = await self.aget(key, default)

    if self._generic_type and data is not None:
        if not self._validate_type(data, self._generic_type):
            from toolboxv2 import get_logger
            get_logger().warning(f"Type mismatch: expected {self._generic_type}, got {type(data)}")

    return data

typed_get(key=None, default=None)

Get data with type validation

Source code in toolboxv2/utils/system/types.py

def typed_get(self, key=None, default=None) -> T:
    """Get data with type validation"""
    data = self.get(key, default)

    if self._generic_type and data is not None:
        # Validate type matches generic parameter
        if not self._validate_type(data, self._generic_type):
            from toolboxv2 import get_logger
            get_logger().warning(f"Type mismatch: expected {self._generic_type}, got {type(data)}")

    return data

typed_json(data, info='OK', interface=ToolBoxInterfaces.remote, exec_code=0, status_code=None) classmethod

Create JSON result with type information

Source code in toolboxv2/utils/system/types.py

@classmethod
def typed_json(cls, data: T, info="OK", interface=ToolBoxInterfaces.remote, exec_code=0,
               status_code=None) -> 'Result[T]':
    """Create JSON result with type information"""
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=status_code or exec_code, help_text=info)

    result = ToolBoxResult(
        data_to=interface,
        data=data,
        data_info="JSON response",
        data_type="json"
    )

    instance = cls(error=error, info=info_obj, result=result)
    if hasattr(cls, '_generic_type'):
        instance._generic_type = cls._generic_type

    return instance

typed_ok(data, data_info='', info='OK', interface=ToolBoxInterfaces.native) classmethod

Create OK result with type information

Source code in toolboxv2/utils/system/types.py

@classmethod
def typed_ok(cls, data: T, data_info="", info="OK", interface=ToolBoxInterfaces.native) -> 'Result[T]':
    """Create OK result with type information"""
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=0, help_text=info)
    result = ToolBoxResult(data_to=interface, data=data, data_info=data_info, data_type=type(data).__name__)

    instance = cls(error=error, info=info_obj, result=result)
    if hasattr(cls, '_generic_type'):
        instance._generic_type = cls._generic_type

    return instance

Singleton

Singleton metaclass for ensuring only one instance of a class.

Source code in toolboxv2/utils/singelton_class.py

class Singleton(type):
    """
    Singleton metaclass for ensuring only one instance of a class.
    """

    _instances = {}
    _kwargs = {}
    _args = {}

    def __call__(cls, *args, **kwargs):
        if cls not in cls._instances:
            cls._instances[cls] = super().__call__(*args, **kwargs)
            cls._args[cls] = args
            cls._kwargs[cls] = kwargs
        return cls._instances[cls]

Spinner

Enhanced Spinner with tqdm-like line rendering.

Source code in toolboxv2/utils/extras/Style.py

class Spinner:
    """
    Enhanced Spinner with tqdm-like line rendering.
    """
    SYMBOL_SETS = {
        "c": ["◐", "◓", "◑", "◒"],
        "b": ["▁", "▃", "▄", "▅", "▆", "▇", "█", "▇", "▆", "▅", "▄", "▃"],
        "d": ["⣾", "⣽", "⣻", "⢿", "⡿", "⣟", "⣯", "⣷"],
        "w": ["🌍", "🌎", "🌏"],
        "s": ["🌀   ", " 🌀  ", "  🌀 ", "   🌀", "  🌀 ", " 🌀  "],
        "+": ["+", "x"],
        "t": ["✶", "✸", "✹", "✺", "✹", "✷"]
    }

    def __init__(
        self,
        message: str = "Loading...",
        delay: float = 0.1,
        symbols=None,
        count_down: bool = False,
        time_in_s: float = 0
    ):
        """Initialize spinner with flexible configuration."""
        # Resolve symbol set.
        if isinstance(symbols, str):
            symbols = self.SYMBOL_SETS.get(symbols, None)

        # Default symbols if not provided.
        if symbols is None:
            symbols = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]

        # Test mode symbol set.
        if 'unittest' in sys.argv[0]:
            symbols = ['#', '=', '-']

        self.spinner = itertools.cycle(symbols)
        self.delay = delay
        self.message = message
        self.running = False
        self.spinner_thread = None
        self.max_t = time_in_s
        self.contd = count_down

        # Rendering management.
        self._is_primary = False
        self._start_time = 0

        # Central manager.
        self.manager = SpinnerManager()

    def _generate_render_line(self):
        """Generate the primary render line."""
        current_time = time.time()
        if self.contd:
            remaining = max(0, self.max_t - (current_time - self._start_time))
            time_display = f"{remaining:.2f}"
        else:
            time_display = f"{current_time - self._start_time:.2f}"

        symbol = next(self.spinner)
        return f"{symbol} {self.message} | {time_display}"

    def _generate_secondary_info(self):
        """Generate secondary spinner info for additional spinners."""
        return f"{self.message}"

    def __enter__(self):
        """Start the spinner."""
        self.running = True
        self._start_time = time.time()
        self.manager.register_spinner(self)
        return self

    def __exit__(self, exc_type, exc_value, exc_traceback):
        """Stop the spinner."""
        self.running = False
        self.manager.unregister_spinner(self)
        # Clear the spinner's line if it was the primary spinner.
        if self._is_primary:
            sys.stdout.write("\r\033[K")
            sys.stdout.flush()

__enter__()

Start the spinner.

Source code in toolboxv2/utils/extras/Style.py

def __enter__(self):
    """Start the spinner."""
    self.running = True
    self._start_time = time.time()
    self.manager.register_spinner(self)
    return self

__exit__(exc_type, exc_value, exc_traceback)

Stop the spinner.

Source code in toolboxv2/utils/extras/Style.py

def __exit__(self, exc_type, exc_value, exc_traceback):
    """Stop the spinner."""
    self.running = False
    self.manager.unregister_spinner(self)
    # Clear the spinner's line if it was the primary spinner.
    if self._is_primary:
        sys.stdout.write("\r\033[K")
        sys.stdout.flush()

__init__(message='Loading...', delay=0.1, symbols=None, count_down=False, time_in_s=0)

Initialize spinner with flexible configuration.

Source code in toolboxv2/utils/extras/Style.py

def __init__(
    self,
    message: str = "Loading...",
    delay: float = 0.1,
    symbols=None,
    count_down: bool = False,
    time_in_s: float = 0
):
    """Initialize spinner with flexible configuration."""
    # Resolve symbol set.
    if isinstance(symbols, str):
        symbols = self.SYMBOL_SETS.get(symbols, None)

    # Default symbols if not provided.
    if symbols is None:
        symbols = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]

    # Test mode symbol set.
    if 'unittest' in sys.argv[0]:
        symbols = ['#', '=', '-']

    self.spinner = itertools.cycle(symbols)
    self.delay = delay
    self.message = message
    self.running = False
    self.spinner_thread = None
    self.max_t = time_in_s
    self.contd = count_down

    # Rendering management.
    self._is_primary = False
    self._start_time = 0

    # Central manager.
    self.manager = SpinnerManager()

TBEF

Automatic generated by ToolBox v = 0.1.22

clis

api

cleanup_build_files()

Clean build artifacts

Source code in toolboxv2/utils/clis/api.py

def cleanup_build_files():
    """Clean build artifacts"""
    print_box_header("Cleaning Build Artifacts", "🧹")
    print_box_footer()

    from toolboxv2 import tb_root_dir
    src_core_path = tb_root_dir / "src-core"
    target_path = src_core_path / "target"

    if target_path.exists():
        try:
            with Spinner("Running cargo clean", symbols="+") as s:
                try:
                    subprocess.run(
                        ["cargo", "clean"],
                        cwd=src_core_path,
                        check=True,
                        capture_output=True
                    )
                except subprocess.CalledProcessError:
                    s.message = "Manually removing build directories"
                    for item in target_path.iterdir():
                        if item.is_dir() and item.name != ".rustc_info.json":
                            shutil.rmtree(item)

            print()
            print_status("Build artifacts cleaned successfully", "success")
            return True

        except Exception as e:
            print()
            print_status(f"Failed to clean build files: {e}", "error")
            return False
    else:
        print_status(f"Build directory not found: {target_path}", "warning")
        return True

cli_api_runner()

Main CLI entry point

Source code in toolboxv2/utils/clis/api.py

def cli_api_runner():
    """Main CLI entry point"""
    parser = argparse.ArgumentParser(
        prog='tb api',
        description='🚀 Platform-Agnostic Rust API Server Manager',
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="""
╔════════════════════════════════════════════════════════════════════════════╗
║                           Command Examples                                 ║
╠════════════════════════════════════════════════════════════════════════════╣
║                                                                            ║
║  Build & Development:                                                      ║
║    $ tb api build                    # Build release executable            ║
║    $ tb api debug                    # Run with hot reload                 ║
║    $ tb api clean                    # Clean build artifacts               ║
║    $ tb api remove-exe               # Remove compiled executable          ║
║    $ tb api visual-test              # Run visual UI component test        ║
║                                                                            ║
║  Server Management:                                                        ║
║    $ tb api start                    # Start server                        ║
║    $ tb api stop                     # Stop server                         ║
║    $ tb api status                   # Check server status                 ║
║                                                                            ║
║  Advanced Options:                                                         ║
║    $ tb api start --posix-zdt        # Start with zero-downtime (Linux)    ║
║    $ tb api start --exe /path --version 1.0.0                              ║
║                                                                            ║
║  Zero-Downtime Updates (Linux/macOS):                                      ║
║    $ tb api update --exe /new/path --version 1.1.0 --posix-zdt             ║
║                                                                            ║
║  Graceful Updates (All Platforms):                                         ║
║    $ tb api update --exe /new/path --version 1.1.0                         ║
║                                                                            ║
╚════════════════════════════════════════════════════════════════════════════╝

Server Configuration:
  Host: {SERVER_HOST}
  Port: {SERVER_PORT}

Zero-Downtime Updates:
  Available on Linux and macOS using --posix-zdt flag
  Uses socket file descriptor passing for seamless updates
  Windows uses graceful restart (brief downtime)
        """
    )

    subparsers = parser.add_subparsers(dest="action", required=False, help="Available actions")

    # Build commands
    subparsers.add_parser('build', help='Build the Rust project in release mode')
    subparsers.add_parser('debug', help='Run in debug mode with hot reload')
    subparsers.add_parser('clean', help='Clean build artifacts')
    subparsers.add_parser('remove-exe', help='Remove release executable')
    subparsers.add_parser('visual-test', help='Run visual test for UI components')

    # Server management commands
    actions = {
        'start': 'Start the API server',
        'stop': 'Stop the running server',
        'update': 'Update server to new version',
        'status': 'Display server status'
    }

    for action, help_text in actions.items():
        p = subparsers.add_parser(action, help=help_text)

        if action in ['start', 'update', 'status']:
            p.add_argument(
                '--posix-zdt',
                action='store_true',
                help='(Linux/macOS) Enable POSIX zero-downtime updates via socket passing'
            )

        if action in ['start', 'update']:
            p.add_argument(
                '--exe',
                type=str,
                help='Path to server executable'
            )
            p.add_argument(
                '--version',
                type=str,
                default='unknown',
                help='Version string for the server'
            )

    args = parser.parse_args()

    # Handle simple actions
    if args.action == 'build':
        handle_build()
        return

    if args.action == 'clean':
        cleanup_build_files()
        return

    if args.action == 'remove-exe':
        remove_release_executable()
        return

    if args.action == 'debug':
        handle_debug()
        return

    if args.action == 'visual-test':
        run_visual_test()
        return

    if not args.action:
        # Default to status if no action provided
        show_server_status(False)
        return

    # Handle server management
    manage_server(
        action=args.action,
        executable_path=getattr(args, 'exe', None),
        version_str=getattr(args, 'version', 'unknown'),
        use_posix_zdt=getattr(args, 'posix_zdt', False)
    )

ensure_socket_and_fd_file_posix(host, port, backlog, fd_file_path)

Create socket and FD file for POSIX zero-downtime updates

Source code in toolboxv2/utils/clis/api.py

def ensure_socket_and_fd_file_posix(host, port, backlog, fd_file_path):
    """Create socket and FD file for POSIX zero-downtime updates"""
    if os.path.exists(fd_file_path):
        print_status(f"Stale FD file found: {fd_file_path}", "warning")
        print_status("Removing to create new socket", "info")
        with contextlib.suppress(OSError):
            os.remove(fd_file_path)

    try:
        print_status("Creating listening socket", "progress")

        server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)

        fd_num = server_socket.fileno()

        # Make socket inheritable
        if hasattr(os, 'set_inheritable'):
            os.set_inheritable(fd_num, True)
        else:
            import fcntl
            flags = fcntl.fcntl(fd_num, fcntl.F_GETFD)
            fcntl.fcntl(fd_num, fcntl.F_SETFD, flags & ~fcntl.FD_CLOEXEC)

        server_socket.bind((host, port))
        server_socket.listen(backlog)

        # Save FD to file
        with open(fd_file_path, 'w') as f:
            f.write(str(fd_num))
        os.chmod(fd_file_path, 0o600)

        print_status(f"Socket created - FD {fd_num} saved to {fd_file_path}", "success")
        return server_socket, fd_num

    except Exception as e:
        print_status(f"Failed to create listening socket: {e}", "error")
        if 'server_socket' in locals():
            server_socket.close()
        return None, None

get_executable_name_with_extension(base_name=DEFAULT_EXECUTABLE_NAME)

Get platform-specific executable name

Source code in toolboxv2/utils/clis/api.py

def get_executable_name_with_extension(base_name=DEFAULT_EXECUTABLE_NAME):
    """Get platform-specific executable name"""
    if platform.system().lower() == "windows":
        return f"{base_name}.exe"
    return base_name

get_executable_path()

Find the release executable in standard locations

Source code in toolboxv2/utils/clis/api.py

def get_executable_path():
    """Find the release executable in standard locations"""
    exe_name = get_executable_name_with_extension()
    from toolboxv2 import tb_root_dir

    search_paths = [
        tb_root_dir / Path("bin") / exe_name,
        tb_root_dir / Path("src-core") / exe_name,
        tb_root_dir / exe_name,
        tb_root_dir / Path("src-core") / "target" / "release" / exe_name,
    ]

    for path in search_paths:
        if path.exists() and path.is_file():
            return path.resolve()

    return None

handle_build()

Build the Rust project

Source code in toolboxv2/utils/clis/api.py

def handle_build():
    """Build the Rust project"""
    print_box_header("Building Rust API Server", "🔨")
    print_box_content("Compiler: Cargo (Rust)", "info")
    print_box_content("Mode: Release", "info")
    print_box_footer()

    from toolboxv2 import tb_root_dir

    try:
        with Spinner("Compiling Rust project", symbols="t") as s:
            result = subprocess.run(
                ["cargo", "build", "--release"],
                cwd=tb_root_dir / "src-core",
                check=True,
                capture_output=True,
                text=True
            )

        print()
        print_status("Build completed successfully", "success")

        # Copy executable
        exe_path = get_executable_path()
        if exe_path:
            bin_dir = tb_root_dir / "bin"
            bin_dir.mkdir(exist_ok=True)

            try:
                dest_path = bin_dir / exe_path.name
                shutil.copy(exe_path, dest_path)
                print_status(f"Executable copied to: {dest_path}", "success")
            except Exception as e:
                print_status(f"Warning: Failed to copy to bin: {e}", "warning")

                # Fallback to ubin
                ubin_dir = tb_root_dir / "ubin"
                ubin_dir.mkdir(exist_ok=True)
                dest_path = ubin_dir / exe_path.name

                try:
                    shutil.copy(exe_path, dest_path)
                    print_status(f"Copied to fallback location: {dest_path}", "info")
                except Exception as e_ubin:
                    print_status(f"Error copying to ubin: {e_ubin}", "error")

    except subprocess.CalledProcessError as e:
        print()
        print_box_header("Build Failed", "✗")
        print_box_content("Compilation errors occurred", "error")
        print_box_footer()
        print("\nError output:")
        print(e.stderr)

    except FileNotFoundError:
        print()
        print_box_header("Build Failed", "✗")
        print_box_content("'cargo' command not found", "error")
        print_box_content("Is Rust installed and in your PATH?", "info")
        print_box_footer()

handle_debug()

Run server in debug mode with hot reload

Source code in toolboxv2/utils/clis/api.py

def handle_debug():
    """Run server in debug mode with hot reload"""
    print_box_header("Debug Mode with Hot Reload", "🔥")
    print_box_content("Watching for file changes", "info")
    print_box_content("Press Ctrl+C to stop", "info")
    print_box_footer()

    from toolboxv2 import tb_root_dir
    src_core_path = tb_root_dir / "src-core"

    # Check if cargo-watch is installed
    try:
        subprocess.run(
            ["cargo", "watch", "--version"],
            check=True,
            cwd=src_core_path,
            capture_output=True
        )
    except Exception:
        print_status("cargo-watch not installed", "warning")
        print_status("Installing cargo-watch...", "progress")

        try:
            with Spinner("Installing cargo-watch", symbols="t"):
                subprocess.run(
                    ["cargo", "install", "cargo-watch"],
                    check=True,
                    cwd=src_core_path,
                    capture_output=True
                )
            print()
            print_status("cargo-watch installed successfully", "success")
        except subprocess.CalledProcessError as e:
            print()
            print_status("Failed to install cargo-watch", "error")
            print_status("Falling back to standard debug mode", "warning")
            print()

            # Fallback to standard debug
            print_separator("═")
            print("  Running in standard debug mode")
            print_separator("═")
            print()

            try:
                subprocess.run(["cargo", "run"], cwd=src_core_path)
            except KeyboardInterrupt:
                print()
                print_status("Debug mode stopped", "info")
            return

    # Run with hot reload
    print_separator("═")
    print("  Starting Hot Reload Session")
    print_separator("═")
    print()

    try:
        subprocess.run(["cargo", "watch", "-x", "run"], cwd=src_core_path)
    except KeyboardInterrupt:
        print()
        print_status("Hot reload stopped", "info")
    except subprocess.CalledProcessError as e:
        print_status(f"Hot reload failed: {e}", "error")

is_process_running(pid)

Check if process is running

Source code in toolboxv2/utils/clis/api.py

def is_process_running(pid):
    """Check if process is running"""
    if pid is None or psutil is None:
        return False
    try:
        return psutil.pid_exists(int(pid))
    except (ValueError, TypeError):
        return False

manage_server(action, executable_path=None, version_str='unknown', use_posix_zdt=False)

Main server management dispatcher

Source code in toolboxv2/utils/clis/api.py

def manage_server(action: str, executable_path: str = None, version_str: str = "unknown", use_posix_zdt: bool = False):
    """Main server management dispatcher"""

    if action == "start":
        if not executable_path:
            executable_path = get_executable_path()

        if not executable_path:
            print_box_header("Executable Not Found", "✗")
            print_box_content("No compiled executable found", "error")
            print_box_content("Build first with: tb api build", "info")
            print_box_footer()
            return False

        start_new_server(executable_path, version_str, use_posix_zdt)

    elif action == "stop":
        pid, _, _ = read_server_state()
        if stop_process(pid):
            write_server_state(None, None, None)

            # Clean up FD file if exists
            is_posix = platform.system().lower() != "windows"
            if is_posix and os.path.exists(PERSISTENT_FD_FILE):
                print()
                print_status(f"Note: Stale FD file exists: {PERSISTENT_FD_FILE}", "warning")
                print_status("Consider removing it before next start", "info")

    elif action == "update":
        if not executable_path:
            print_box_header("Update Failed", "✗_")
            print_box_content("New executable path required (--exe)", "error")
            print_box_footer()
            return False

        if not version_str or version_str == "unknown":
            print_box_header("Update Failed", "✗_")
            print_box_content("Version string required (--version)", "error")
            print_box_footer()
            return False

        update_server(executable_path, version_str, use_posix_zdt)

    elif action == "status":
        show_server_status(use_posix_zdt)

    return True

read_server_state(state_file=SERVER_STATE_FILE)

Read server state from file

Source code in toolboxv2/utils/clis/api.py

def read_server_state(state_file=SERVER_STATE_FILE):
    """Read server state from file"""
    try:
        if os.path.exists(state_file):
            with open(state_file) as f:
                state = json.load(f)
                return state.get('pid'), state.get('version'), state.get('executable_path')
        return None, None, None
    except Exception:
        return None, None, None

remove_release_executable()

Remove release executable

Source code in toolboxv2/utils/clis/api.py

def remove_release_executable():
    """Remove release executable"""
    print_box_header("Removing Release Executable", "🗑️")
    print_box_footer()

    from toolboxv2 import tb_root_dir
    src_core_path = tb_root_dir / "src-core"
    exe_name = get_executable_name_with_extension()

    paths_to_remove = [
        src_core_path / exe_name,
        src_core_path / "target" / "release" / exe_name
    ]

    removed_count = 0

    for path in paths_to_remove:
        if path.exists():
            try:
                path.unlink()
                print_status(f"Removed: {path}", "success")
                removed_count += 1
            except Exception as e:
                print_status(f"Failed to remove {path}: {e}", "error")

    if removed_count == 0:
        print_status("No executables found to remove", "warning")
    else:
        print()
        print_status(f"Removed {removed_count} executable(s)", "success")

    return True

show_server_status(use_posix_zdt)

Display current server status

Source code in toolboxv2/utils/clis/api.py

def show_server_status(use_posix_zdt: bool):
    """Display current server status"""
    pid, ver, exe = read_server_state()

    print_box_header("Server Status", "🖥️")
    print()

    if is_process_running(pid):
        # Running - show full details
        columns = [
            ("Property", 15),
            ("Value", 50)
        ]
        widths = [w for _, w in columns]

        print_table_header(columns, widths)
        print_table_row(["Status", "RUNNING"], widths, ["white", "green"])
        print_table_row(["PID", str(pid)], widths, ["white", "grey"])
        print_table_row(["Version", ver or 'N/A'], widths, ["white", "yellow"])
        print_table_row(["Executable", str(exe) if exe else 'N/A'], widths, ["white", "cyan"])
        print_table_row(["Host", SERVER_HOST], widths, ["white", "blue"])
        print_table_row(["Port", str(SERVER_PORT)], widths, ["white", "blue"])

        # Check for POSIX ZDT
        is_posix = platform.system().lower() != "windows"
        if is_posix and os.path.exists(PERSISTENT_FD_FILE) and use_posix_zdt:
            try:
                with open(PERSISTENT_FD_FILE) as f:
                    fd_val = f.read().strip()
                print_table_row(["ZDT Mode", f"Active (FD: {fd_val})"], widths, ["white", "green"])
            except Exception:
                pass

        print()
        print_status("Server is healthy and running", "success")

    else:
        print_box_content("Status: STOPPED", "error")
        if pid:
            print_box_content(f"Stale PID in state: {pid}", "warning")
        print()
        print_status("Server is not running", "warning")

    print_box_footer()

start_new_server(executable_path, version_str, use_posix_zdt)

Start a new server instance

Source code in toolboxv2/utils/clis/api.py

def start_new_server(executable_path, version_str, use_posix_zdt):
    """Start a new server instance"""
    current_pid, _, _ = read_server_state()

    if is_process_running(current_pid):
        print_box_header("Server Already Running", "⚠")
        print_box_content(f"PID: {current_pid}", "warning")
        print_box_content("Use 'stop' or 'update' command", "info")
        print_box_footer()
        return

    print_box_header(f"Starting Server v{version_str}", "🚀")
    print_box_content(f"Executable: {executable_path}", "info")
    print_box_content(f"Host: {SERVER_HOST}:{SERVER_PORT}", "network")

    is_posix = platform.system().lower() != "windows"

    if is_posix and use_posix_zdt:
        print_box_content("Mode: POSIX Zero-Downtime", "info")
    else:
        print_box_content("Mode: Standard Start", "info")

    print_box_footer()

    process = None
    socket_obj = None

    with Spinner(f"Launching server", symbols="d") as s:
        if is_posix and use_posix_zdt:
            socket_obj, fd = ensure_socket_and_fd_file_posix(
                SERVER_HOST, SERVER_PORT, SOCKET_BACKLOG, PERSISTENT_FD_FILE
            )
            if fd is not None:
                process = start_rust_server_posix(executable_path, fd)
        else:
            process = start_rust_server_windows(executable_path)

        time.sleep(2)  # Stabilization period

    # Close parent's socket handle
    if socket_obj:
        socket_obj.close()

    print()

    if process and process.poll() is None:
        write_server_state(process.pid, version_str, executable_path)

        print_box_header("Server Started", "✓")
        print_box_content(f"Version: {version_str}", "success")
        print_box_content(f"PID: {process.pid}", "success")
        print_box_content(f"Port: {SERVER_PORT}", "success")
        print_box_footer()
    else:
        print_box_header("Server Failed to Start", "✗")
        print_box_content("Check logs for details", "error")
        print_box_footer()
        write_server_state(None, None, None)

start_rust_server_posix(executable_path, persistent_fd)

Start Rust server on POSIX with socket FD passing

Source code in toolboxv2/utils/clis/api.py

def start_rust_server_posix(executable_path: str, persistent_fd: int):
    """Start Rust server on POSIX with socket FD passing"""
    abs_path = Path(executable_path).resolve()

    env = os.environ.copy()
    env["PERSISTENT_LISTENER_FD"] = str(persistent_fd)

    print_status(f"Starting {abs_path.name} with FD {persistent_fd}", "server")

    try:
        return subprocess.Popen(
            [str(abs_path)],
            cwd=abs_path.parent,
            env=env,
            pass_fds=[persistent_fd]
        )
    except Exception as e:
        print_status(f"Failed to start server: {e}", "error")
        return None

start_rust_server_windows(executable_path)

Start Rust server on Windows

Source code in toolboxv2/utils/clis/api.py

def start_rust_server_windows(executable_path: str):
    """Start Rust server on Windows"""
    abs_path = Path(executable_path).resolve()

    print_status(f"Starting {abs_path.name}", "server")
    print_status(f"Working directory: {abs_path.parent}", "info")

    try:
        return subprocess.Popen([str(abs_path)], cwd=abs_path.parent)
    except Exception as e:
        print_status(f"Failed to start server: {e}", "error")
        return None

stop_process(pid, timeout=10)

Stop process gracefully with timeout

Source code in toolboxv2/utils/clis/api.py

def stop_process(pid, timeout=10):
    """Stop process gracefully with timeout"""
    if not is_process_running(pid):
        print_status(f"Process {pid} not running", "warning")
        return True

    print_box_header(f"Stopping Process", "⏹️")
    print_box_content(f"PID: {pid}", "info")
    print_box_content(f"Timeout: {timeout}s", "info")
    print_box_footer()

    with Spinner(f"Stopping process {pid}", symbols="+", time_in_s=timeout, count_down=True) as s:
        try:
            proc = psutil.Process(int(pid))
            proc.terminate()
            proc.wait(timeout)
        except psutil.TimeoutExpired:
            s.message = f"Force killing process {pid}"
            proc.kill()
        except psutil.NoSuchProcess:
            pass
        except Exception as e:
            print()
            print_status(f"Error stopping process {pid}: {e}", "error")
            return False

    print()
    print_status(f"Process {pid} stopped successfully", "success")
    return True

update_server(new_executable_path, new_version, use_posix_zdt)

High-level update function

Source code in toolboxv2/utils/clis/api.py

def update_server(new_executable_path: str, new_version: str, use_posix_zdt: bool):
    """High-level update function"""
    is_posix = platform.system().lower() != "windows"

    if is_posix and use_posix_zdt:
        return update_server_posix(new_executable_path, new_version)
    else:
        if use_posix_zdt and not is_posix:
            print_status("--posix-zdt flag ignored on Windows", "warning")
            print_status("Using graceful restart instead", "info")
            print()
        return update_server_graceful_restart(new_executable_path, new_version)

update_server_graceful_restart(new_executable_path, new_version)

Perform graceful restart update

Source code in toolboxv2/utils/clis/api.py

def update_server_graceful_restart(new_executable_path: str, new_version: str):
    """Perform graceful restart update"""
    print_box_header(f"Graceful Restart to v{new_version}", "🔄")
    print_box_content("Method: Stop & Start", "info")
    print_box_footer()

    old_pid, old_version, _ = read_server_state()

    # Step 1: Stop old server
    print_separator("═")
    print("  PHASE 1: Stopping Current Server")
    print_separator("═")
    print()

    if not stop_process(old_pid):
        print_status("Failed to stop old server", "error")
        print_status("Update aborted to prevent conflicts", "error")
        return False

    # Step 2: Start new server
    print()
    print_separator("═")
    print("  PHASE 2: Starting New Server")
    print_separator("═")
    print()

    start_new_server(new_executable_path, new_version, False)

    return True

update_server_posix(new_executable_path, new_version)

Perform POSIX zero-downtime update

Source code in toolboxv2/utils/clis/api.py

def update_server_posix(new_executable_path: str, new_version: str):
    """Perform POSIX zero-downtime update"""
    print_box_header(f"Zero-Downtime Update to v{new_version}", "🔄")
    print_box_content("Method: POSIX Socket Passing", "info")
    print_box_footer()

    old_pid, old_version, _ = read_server_state()

    if not os.path.exists(PERSISTENT_FD_FILE):
        print_status(f"FD file '{PERSISTENT_FD_FILE}' not found", "error")
        print_status("Cannot perform zero-downtime update", "error")
        return False

    try:
        with open(PERSISTENT_FD_FILE) as f:
            persistent_fd = int(f.read().strip())
        print_status(f"Using FD {persistent_fd} for socket passing", "info")
    except Exception as e:
        print_status(f"Error reading FD from file: {e}", "error")
        return False

    # Step 1: Start new server
    print()
    print_separator("═")
    print("  PHASE 1: Starting New Server")
    print_separator("═")
    print()

    with Spinner(f"Starting new server v{new_version}", symbols="d") as s:
        new_process = start_rust_server_posix(new_executable_path, persistent_fd)
        time.sleep(3)

    print()

    if new_process is None or new_process.poll() is not None:
        print_status("New server process died on startup", "error")
        print_status("Update aborted", "error")
        return False

    print_status(f"New server started (PID: {new_process.pid})", "success")

    # Step 2: Stop old server
    print()
    print_separator("═")
    print("  PHASE 2: Stopping Old Server")
    print_separator("═")
    print()

    if stop_process(old_pid):
        write_server_state(new_process.pid, new_version, new_executable_path)

        print()
        print_box_header("Update Complete", "✓")
        print_box_content(f"Old Version: {old_version} (PID: {old_pid})", "info")
        print_box_content(f"New Version: {new_version} (PID: {new_process.pid})", "success")
        print_box_content("Zero downtime achieved!", "success")
        print_box_footer()
        return True
    else:
        print_status("Failed to stop old process", "error")
        print_status("Manual intervention may be required", "warning")
        stop_process(new_process.pid)
        return False

write_server_state(pid, server_version, executable_path, state_file=SERVER_STATE_FILE)

Write server state to file

Source code in toolboxv2/utils/clis/api.py

def write_server_state(pid, server_version, executable_path, state_file=SERVER_STATE_FILE):
    """Write server state to file"""
    if executable_path is None:
        executable_path = ''
    try:
        state = {
            'pid': pid,
            'version': server_version,
            'executable_path': str(Path(executable_path).resolve())
        }
        with open(state_file, 'w') as f:
            json.dump(state, f, indent=4)
    except Exception as e:
        print_status(f"Error writing server state: {e}", "error")

cli_printing

Colors

ANSI color codes for terminal styling

Source code in toolboxv2/utils/clis/cli_printing.py

class Colors:
    """ANSI color codes for terminal styling"""
    # Basic colors
    BLACK = '\033[30m'
    RED = '\033[31m'
    GREEN = '\033[32m'
    YELLOW = '\033[33m'
    BLUE = '\033[34m'
    MAGENTA = '\033[35m'
    CYAN = '\033[36m'
    WHITE = '\033[37m'
    GREY = '\033[90m'

    # Bright colors
    BRIGHT_RED = '\033[91m'
    BRIGHT_GREEN = '\033[92m'
    BRIGHT_YELLOW = '\033[93m'
    BRIGHT_BLUE = '\033[94m'
    BRIGHT_MAGENTA = '\033[95m'
    BRIGHT_CYAN = '\033[96m'
    BRIGHT_WHITE = '\033[97m'

    # Styles
    BOLD = '\033[1m'
    DIM = '\033[2m'
    ITALIC = '\033[3m'
    UNDERLINE = '\033[4m'
    BLINK = '\033[5m'
    REVERSE = '\033[7m'

    # Background colors
    BG_BLACK = '\033[40m'
    BG_RED = '\033[41m'
    BG_GREEN = '\033[42m'
    BG_YELLOW = '\033[43m'
    BG_BLUE = '\033[44m'
    BG_MAGENTA = '\033[45m'
    BG_CYAN = '\033[46m'
    BG_WHITE = '\033[47m'

    # Reset
    RESET = '\033[0m'

main()

Entry point for running visual test directly

Source code in toolboxv2/utils/clis/cli_printing.py

def main():
    """Entry point for running visual test directly"""
    run_visual_test()

print_box_content(text, style='', width=76, auto_wrap=True)

Print content with minimal styled prefix

Source code in toolboxv2/utils/clis/cli_printing.py

def print_box_content(text: str, style: str = "", width: int = 76, auto_wrap: bool = True):
    """Print content with minimal styled prefix"""
    style_config = {
        'success': {'icon': '✓', 'color': Colors.GREEN},
        'error': {'icon': '✗', 'color': Colors.RED},
        'warning': {'icon': '⚠', 'color': Colors.YELLOW},
        'info': {'icon': 'ℹ', 'color': Colors.BLUE},
    }

    if style in style_config:
        config = style_config[style]
        print(f"  {config['color']}{config['icon']}{Colors.RESET} {text}")
    else:
        print(f"  {text}")

print_box_footer(width=76)

Print a minimal footer

Source code in toolboxv2/utils/clis/cli_printing.py

def print_box_footer(width: int = 76):
    """Print a minimal footer"""
    print()

print_box_header(title, icon='ℹ', width=76)

Print a minimal styled header

Source code in toolboxv2/utils/clis/cli_printing.py

def print_box_header(title: str, icon: str = "ℹ", width: int = 76):
    """Print a minimal styled header"""
    print()
    print(f"{Colors.BOLD}{icon} {title}{Colors.RESET}")
    print(f"{Colors.DIM}{'─' * width}{Colors.RESET}")

print_code_block(code, language='text', width=76, show_line_numbers=False)

Print code block with minimal syntax highlighting

Source code in toolboxv2/utils/clis/cli_printing.py

def print_code_block(code: str, language: str = "text", width: int = 76, show_line_numbers: bool = False):
    """Print code block with minimal syntax highlighting"""
    import json

    if language.lower() in ['json']:
        try:
            parsed = json.loads(code) if isinstance(code, str) else code
            formatted = json.dumps(parsed, indent=2)
            lines = formatted.split('\n')
        except:
            lines = code.split('\n')
    elif language.lower() in ['yaml', 'yml']:
        lines = code.split('\n')
        formatted_lines = []
        for line in lines:
            if ':' in line and not line.strip().startswith('#'):
                key, value = line.split(':', 1)
                formatted_lines.append(f"{Colors.CYAN}{key}{Colors.RESET}:{value}")
            elif line.strip().startswith('#'):
                formatted_lines.append(f"{Colors.DIM}{line}{Colors.RESET}")
            else:
                formatted_lines.append(line)
        lines = formatted_lines
    elif language.lower() in ['toml']:
        lines = code.split('\n')
        formatted_lines = []
        for line in lines:
            if line.strip().startswith('[') and line.strip().endswith(']'):
                formatted_lines.append(f"{Colors.BOLD}{line}{Colors.RESET}")
            elif '=' in line and not line.strip().startswith('#'):
                key, value = line.split('=', 1)
                formatted_lines.append(f"{Colors.CYAN}{key}{Colors.RESET}={value}")
            elif line.strip().startswith('#'):
                formatted_lines.append(f"{Colors.DIM}{line}{Colors.RESET}")
            else:
                formatted_lines.append(line)
        lines = formatted_lines
    elif language.lower() in ['env', 'dotenv']:
        lines = code.split('\n')
        formatted_lines = []
        for line in lines:
            if '=' in line and not line.strip().startswith('#'):
                key, value = line.split('=', 1)
                formatted_lines.append(f"{Colors.CYAN}{key}{Colors.RESET}={value}")
            elif line.strip().startswith('#'):
                formatted_lines.append(f"{Colors.DIM}{line}{Colors.RESET}")
            else:
                formatted_lines.append(line)
        lines = formatted_lines
    else:
        lines = code.split('\n')

    for i, line in enumerate(lines, 1):
        if show_line_numbers:
            print(f"  {Colors.DIM}{i:3d}{Colors.RESET} {line}")
        else:
            print(f"  {line}")

print_separator(char='─', width=76)

Print a minimal separator line

Source code in toolboxv2/utils/clis/cli_printing.py

def print_separator(char: str = "─", width: int = 76):
    """Print a minimal separator line"""
    print(f"{Colors.DIM}{char * width}{Colors.RESET}")

print_status(message, status='info')

Print a minimal status message with icon and color

Source code in toolboxv2/utils/clis/cli_printing.py

def print_status(message: str, status: str = "info"):
    """Print a minimal status message with icon and color"""
    status_config = {
        'success': {'icon': '✓', 'color': Colors.GREEN},
        'error': {'icon': '✗', 'color': Colors.RED},
        'warning': {'icon': '⚠', 'color': Colors.YELLOW},
        'info': {'icon': 'ℹ', 'color': Colors.BLUE},
        'progress': {'icon': '⟳', 'color': Colors.CYAN},
        'waiting': {'icon': '⏳', 'color': Colors.MAGENTA},
        'launch': {'icon': '🚀', 'color': Colors.GREEN},
        'install': {'icon': '📦', 'color': Colors.CYAN},
        'download': {'icon': '⬇️', 'color': Colors.BLUE},
        'upload': {'icon': '⬆️', 'color': Colors.MAGENTA},
        'connect': {'icon': '🔗', 'color': Colors.GREEN},
        'disconnect': {'icon': '🔌', 'color': Colors.RED},
        'configure': {'icon': '🔧', 'color': Colors.YELLOW},
        'debug': {'icon': '🐞', 'color': Colors.MAGENTA},
        'test': {'icon': '🧪', 'color': Colors.GREEN},
        'analyze': {'icon': '🔍', 'color': Colors.BLUE},
        'data': {'icon': '💾', 'color': Colors.YELLOW},
        'database': {'icon': '🗃️', 'color': Colors.MAGENTA},
        'server': {'icon': '🖥️', 'color': Colors.GREEN},
        'network': {'icon': '🌐', 'color': Colors.BLUE},
        'build': {'icon': '🔨', 'color': Colors.CYAN},
        'update': {'icon': '🔄', 'color': Colors.MAGENTA}
    }

    config = status_config.get(status, {'icon': '•', 'color': ''})

    print(f"{config['color']}{config['icon']}{Colors.RESET} {message}")

print_table_header(columns, widths)

Print a table header with columns

Source code in toolboxv2/utils/clis/cli_printing.py

def print_table_header(columns: list, widths: list):
    """Print a table header with columns"""
    header_parts = []
    for (name, _), width in zip(columns, widths):
        header_parts.append(f"{Colors.BOLD}{Colors.BRIGHT_WHITE}{name:<{width}}{Colors.RESET}")

    print(f"  {' │ '.join(header_parts)}")

    sep_parts = [f"{Colors.BRIGHT_CYAN}{'─' * w}{Colors.RESET}" for w in widths]
    print(f"  {f'{Colors.BRIGHT_CYAN}─┼─{Colors.RESET}'.join(sep_parts)}")

print_table_row(values, widths, styles=None)

Print a table row

Source code in toolboxv2/utils/clis/cli_printing.py

def print_table_row(values: list, widths: list, styles: list = None):
    """Print a table row"""
    if styles is None:
        styles = [""] * len(values)

    color_map = {
        'grey': Colors.GREY,
        'white': Colors.WHITE,
        'green': Colors.BRIGHT_GREEN,
        'yellow': Colors.BRIGHT_YELLOW,
        'cyan': Colors.BRIGHT_CYAN,
        'blue': Colors.BRIGHT_BLUE,
        'red': Colors.BRIGHT_RED,
        'magenta': Colors.BRIGHT_MAGENTA,
    }

    row_parts = []
    for value, width, style in zip(values, widths, styles):
        color = color_map.get(style.lower(), '')
        if color:
            colored_value = f"{color}{value}{Colors.RESET}"
            padding = width - len(value)
            row_parts.append(colored_value + " " * padding)
        else:
            row_parts.append(f"{value:<{width}}")

    print(f"  {f' {Colors.DIM}│{Colors.RESET} '.join(row_parts)}")

run_visual_test()

Visual test for all UI components - for alignment and testing

Source code in toolboxv2/utils/clis/cli_printing.py

def run_visual_test():
    """Visual test for all UI components - for alignment and testing"""
    print("\n" + f"{Colors.BOLD}{Colors.BRIGHT_CYAN}{'=' * 80}{Colors.RESET}")
    print(f"{Colors.BOLD}{Colors.BRIGHT_WHITE} VISUAL TEST - CLI UI COMPONENTS ".center(80, '='))
    print(f"{Colors.BOLD}{Colors.BRIGHT_CYAN}{'=' * 80}{Colors.RESET}\n")

    # Test 1: Headers with different icons
    print(f"{Colors.BOLD}TEST 1: Headers with Different Icons{Colors.RESET}")
    print(f"{Colors.DIM}{'-' * 80}{Colors.RESET}")

    print_box_header("Standard Info Header", "ℹ")
    print_box_footer()

    print_box_header("Success Header", "✓")
    print_box_footer()

    print_box_header("Server Header", "🖥️")
    print_box_footer()

    # Test 2: Content with different styles
    print(f"\n{Colors.BOLD}TEST 2: Content with Different Styles{Colors.RESET}")
    print(f"{Colors.DIM}{'-' * 80}{Colors.RESET}")
    print_box_header("Content Styles Test", "🎨")
    print_box_content("This is a plain text without style")
    print_box_content("This is a success message", "success")
    print_box_content("This is an error message", "error")
    print_box_content("This is a warning message", "warning")
    print_box_content("This is an info message", "info")
    print_box_footer()

    # Test 3: Combined content
    print(f"\n{Colors.BOLD}TEST 3: Combined Content{Colors.RESET}")
    print(f"{Colors.DIM}{'-' * 80}{Colors.RESET}")
    print_box_header("Server Status Example", "🖥️")
    print_box_content("Server Name: ToolBoxV2 API Server", "info")
    print_box_content("Status: Running", "success")
    print_box_content("Port: 8080", "info")
    print_box_content("Warning: High memory usage detected", "warning")
    print_box_content("Error: Connection timeout on endpoint /api/test", "error")
    print_box_content("Plain text information line")
    print_box_footer()

    # Test 4: Status messages
    print(f"\n{Colors.BOLD}TEST 4: Status Messages{Colors.RESET}")
    print(f"{Colors.DIM}{'-' * 80}{Colors.RESET}")
    print_status("Success status message", "success")
    print_status("Error status message", "error")
    print_status("Warning status message", "warning")
    print_status("Info status message", "info")
    print_status("Progress status message", "progress")
    print_status("Server status message", "server")
    print_status("Build status message", "build")
    print_status("Update status message", "update")

    # Test 5: Separators
    print(f"\n{Colors.BOLD}TEST 5: Separators{Colors.RESET}")
    print(f"{Colors.DIM}{'-' * 80}{Colors.RESET}")
    print_separator("─")
    print_separator("═")
    print_separator("━")

    # Test 6: Tables
    print(f"\n{Colors.BOLD}TEST 6: Table Display{Colors.RESET}")
    print(f"{Colors.DIM}{'-' * 80}{Colors.RESET}")
    columns = [
        ("Property", 20),
        ("Value", 30),
        ("Status", 20)
    ]
    widths = [w for _, w in columns]

    print_table_header(columns, widths)
    print_table_row(["Server Name", "ToolBoxV2 API", "Active"], widths, ["white", "cyan", "green"])
    print_table_row(["PID", "12345", "Running"], widths, ["white", "grey", "green"])
    print_table_row(["Version", "1.0.0", "Latest"], widths, ["white", "yellow", "green"])
    print_table_row(["Port", "8080", "Open"], widths, ["white", "blue", "green"])

    # Test 7: Code blocks
    print(f"\n\n{Colors.BOLD}TEST 7: Code & Config File Display{Colors.RESET}")
    print(f"{Colors.DIM}{'-' * 80}{Colors.RESET}")

    print_box_header("JSON Configuration", "📄")
    json_example = '''{
  "server": {
    "host": "0.0.0.0",
    "port": 8080,
    "debug": true
  },
  "database": {
    "url": "postgresql://localhost/mydb",
    "pool_size": 10
  }
}'''
    print_code_block(json_example, "json", show_line_numbers=True)
    print_box_footer()

    print_box_header("Environment Variables", "📄")
    env_example = '''# Application Settings
APP_NAME=ToolBoxV2
APP_ENV=production
DEBUG=false

# Database
DATABASE_URL=postgresql://localhost/mydb'''
    print_code_block(env_example, "env")
    print_box_footer()

    # Test 8: Real-world example
    print(f"\n{Colors.BOLD}TEST 8: Real-World Server Start Example{Colors.RESET}")
    print(f"{Colors.DIM}{'-' * 80}{Colors.RESET}")
    print_box_header("Starting Server v1.2.3", "🚀")
    print_box_content("Executable: /usr/local/bin/simple-core-server", "info")
    print_box_content("Host: 0.0.0.0:8080", "info")
    print_box_content("Mode: POSIX Zero-Downtime", "info")
    print_box_footer()

    print_status("Launching server", "progress")
    print_status("Socket created - FD 3 saved to server_socket.fd", "success")
    print()

    print_box_header("Server Started", "✓")
    print_box_content("Version: 1.2.3", "success")
    print_box_content("PID: 54321", "success")
    print_box_content("Port: 8080", "success")
    print_box_footer()

    print(f"\n{Colors.BOLD}{Colors.BRIGHT_CYAN}{'=' * 80}{Colors.RESET}")
    print(f"{Colors.BOLD}{Colors.BRIGHT_WHITE} END OF VISUAL TEST ".center(80, '='))
    print(f"{Colors.BOLD}{Colors.BRIGHT_CYAN}{'=' * 80}{Colors.RESET}\n")

db_cli_manager

ClusterManager

Manages a cluster of r_blob_db instances defined in a config file.

Source code in toolboxv2/utils/clis/db_cli_manager.py

class ClusterManager:
    """Manages a cluster of r_blob_db instances defined in a config file."""

    def __init__(self, config_path: str = CLUSTER_CONFIG_FILE):
        self.config_path = Path(config_path)
        self.instances: dict[str, DBInstanceManager] = self._load_config()

    def _load_config(self) -> dict[str, DBInstanceManager]:
        """Loads and validates the cluster configuration."""
        from toolboxv2 import tb_root_dir
        if not self.config_path.is_absolute():
            self.config_path = tb_root_dir / self.config_path

        default_config_dir = (tb_root_dir / ".data/db_data/").resolve()
        default_config = {
            "instance-01": {"port": 3001, "data_dir": str(default_config_dir / "01")},
            "instance-02": {"port": 3002, "data_dir": str(default_config_dir / "02")},
        }

        if not self.config_path.exists():
            print_status(f"Cluster config '{self.config_path}' not found. Creating default", "warning")

            with open(self.config_path, 'w') as f:
                json.dump(default_config, f, indent=4)
            config_data = default_config
        else:
            try:
                with open(self.config_path) as f:
                    config_data = json.load(f)
            except json.JSONDecodeError:
                print_status(f"Cluster config '{self.config_path}' is not valid JSON. Using default", "error")
                config_data = default_config

        return {id: DBInstanceManager(id, cfg) for id, cfg in config_data.items()}

    def get_instances(self, instance_id: str | None = None) -> list[DBInstanceManager]:
        """Returns a list of instances to operate on."""
        if instance_id:
            if instance_id not in self.instances:
                raise ValueError(f"Instance ID '{instance_id}' not found in '{self.config_path}'.")
            return [self.instances[instance_id]]
        return list(self.instances.values())

    def start_all(self, executable_path: Path, version: str, instance_id: str | None = None):
        """Start all instances"""
        instances = self.get_instances(instance_id)

        if len(instances) > 1:
            print_box_header("Starting Multiple Instances", "🚀")
            print_box_content(f"Total instances: {len(instances)}", "info")
            print_box_footer()

        for instance in instances:
            instance.start(executable_path, version)
            if len(instances) > 1:
                print()

    def stop_all(self, instance_id: str | None = None):
        """Stop all instances"""
        instances = self.get_instances(instance_id)

        if len(instances) > 1:
            print_box_header("Stopping Multiple Instances", "⏹️")
            print_box_content(f"Total instances: {len(instances)}", "info")
            print_box_footer()

        for instance in instances:
            instance.stop()
            if len(instances) > 1:
                print()

    def status_all(self, instance_id: str | None = None, silent=False):
        """Show status of all instances"""
        instances = self.get_instances(instance_id)

        if not silent:

            # Table header
            columns = [
                ("INSTANCE ID", 18),
                ("STATUS", 12),
                ("PID", 8),
                ("VERSION", 12),
                ("PORT", 6),
                ("HOST", 15)
            ]
            widths = [w for _, w in columns]

            print("🖥️ Cluster Status\n")
            print_table_header(columns, widths)

        services_online = 0
        server_list = []

        for instance in instances:
            pid, version = instance.read_state()
            is_running = instance.is_running()

            if is_running:
                server_list.append(f"http://{instance.host}:{instance.port}")
                services_online += 1

            if not silent:
                status_str = " RUNNING" if is_running else " STOPPED"
                status_style = "green" if is_running else "red"

                print_table_row(
                    [
                        instance.id,
                        status_str,
                        str(pid or 'N/A'),
                        version or 'N/A',
                        str(instance.port),
                        instance.host
                    ],
                    widths,
                    ["white", status_style, "grey", "blue", "yellow", "cyan"]
                )

        if not silent:
            print()
            print_status(f"Services online: {services_online}/{len(instances)}", "info")

        return services_online, server_list

    def health_check_all(self, instance_id: str | None = None):
        """Perform health check on all instances"""
        instances = self.get_instances(instance_id)

        print("🏥 Cluster Health Check\n")

        columns = [
            ("INSTANCE ID", 18),
            ("STATUS", 12),
            ("PID", 8),
            ("LATENCY", 10),
            ("BLOBS", 8),
            ("VERSION", 12)
        ]
        widths = [w for _, w in columns]
        print_table_header(columns, widths)

        healthy_count = 0

        for instance in instances:
            health = instance.get_health()
            status = health.get('status', 'UNKNOWN')
            pid = health.get('pid', 'N/A')

            if status == 'OK':
                healthy_count += 1
                status_str, style = " OK", "green"
                latency = f"{health['latency_ms']}ms"
                blobs = str(health.get('blobs_managed', 'N/A'))
                version = health.get('server_version', 'N/A')
            elif status == 'STOPPED':
                status_str, style = "❌ STOPPED", "red"
                latency = blobs = version = "N/A"
            else:
                status_str, style = f"🔥 {status}", "red"
                latency = blobs = version = "N/A"

            print_table_row(
                [instance.id, status_str, str(pid), latency, blobs, version],
                widths,
                ["white", style, "grey", "green", "yellow", "blue"]
            )

        print()
        print_status(f"Healthy instances: {healthy_count}/{len(instances)}",
                     "success" if healthy_count == len(instances) else "warning")

    def update_all_rolling(self, new_executable_path: Path, new_version: str, instance_id: str | None = None):
        """Performs a zero-downtime rolling update of the cluster."""
        instances_to_update = self.get_instances(instance_id)

        print_box_header(f"Rolling Update to Version {new_version}", "🔄")
        print_box_content(f"Instances to update: {len(instances_to_update)}", "info")
        print_box_content(f"Executable: {new_executable_path}", "info")
        print_box_footer()

        for i, instance in enumerate(instances_to_update):
            print_separator("═")
            print(f"  [{i + 1}/{len(instances_to_update)}] Updating instance '{instance.id}'")
            print_separator("═")
            print()

            if not instance.stop():
                print_status(f"CRITICAL: Failed to stop old instance '{instance.id}'. Aborting", "error")
                return

            if not instance.start(new_executable_path, new_version):
                print_status(f"CRITICAL: Failed to start new version for '{instance.id}'", "error")
                print_status("The cluster might be in a partially updated state", "warning")
                return

            # Health check with retries
            print()
            with Spinner(f"Waiting for '{instance.id}' to become healthy", symbols="t") as s:
                for attempt in range(5):
                    s.message = f"Waiting for '{instance.id}' to become healthy (attempt {attempt + 1}/5)"
                    time.sleep(2)
                    health = instance.get_health()
                    if health.get('status') == 'OK':
                        print()
                        print_status(f"Instance '{instance.id}' is healthy with new version", "success")
                        break
                else:
                    print()
                    print_status(f"Instance '{instance.id}' did not become healthy. Update halted", "error")
                    return
            print()

        print_separator("═")
        print_status("Rolling Update Complete!", "success")
        print_separator("═")

get_instances(instance_id=None)

Returns a list of instances to operate on.

Source code in toolboxv2/utils/clis/db_cli_manager.py

def get_instances(self, instance_id: str | None = None) -> list[DBInstanceManager]:
    """Returns a list of instances to operate on."""
    if instance_id:
        if instance_id not in self.instances:
            raise ValueError(f"Instance ID '{instance_id}' not found in '{self.config_path}'.")
        return [self.instances[instance_id]]
    return list(self.instances.values())

health_check_all(instance_id=None)

Perform health check on all instances

Source code in toolboxv2/utils/clis/db_cli_manager.py

def health_check_all(self, instance_id: str | None = None):
    """Perform health check on all instances"""
    instances = self.get_instances(instance_id)

    print("🏥 Cluster Health Check\n")

    columns = [
        ("INSTANCE ID", 18),
        ("STATUS", 12),
        ("PID", 8),
        ("LATENCY", 10),
        ("BLOBS", 8),
        ("VERSION", 12)
    ]
    widths = [w for _, w in columns]
    print_table_header(columns, widths)

    healthy_count = 0

    for instance in instances:
        health = instance.get_health()
        status = health.get('status', 'UNKNOWN')
        pid = health.get('pid', 'N/A')

        if status == 'OK':
            healthy_count += 1
            status_str, style = " OK", "green"
            latency = f"{health['latency_ms']}ms"
            blobs = str(health.get('blobs_managed', 'N/A'))
            version = health.get('server_version', 'N/A')
        elif status == 'STOPPED':
            status_str, style = "❌ STOPPED", "red"
            latency = blobs = version = "N/A"
        else:
            status_str, style = f"🔥 {status}", "red"
            latency = blobs = version = "N/A"

        print_table_row(
            [instance.id, status_str, str(pid), latency, blobs, version],
            widths,
            ["white", style, "grey", "green", "yellow", "blue"]
        )

    print()
    print_status(f"Healthy instances: {healthy_count}/{len(instances)}",
                 "success" if healthy_count == len(instances) else "warning")

start_all(executable_path, version, instance_id=None)

Start all instances

Source code in toolboxv2/utils/clis/db_cli_manager.py

def start_all(self, executable_path: Path, version: str, instance_id: str | None = None):
    """Start all instances"""
    instances = self.get_instances(instance_id)

    if len(instances) > 1:
        print_box_header("Starting Multiple Instances", "🚀")
        print_box_content(f"Total instances: {len(instances)}", "info")
        print_box_footer()

    for instance in instances:
        instance.start(executable_path, version)
        if len(instances) > 1:
            print()

status_all(instance_id=None, silent=False)

Show status of all instances

Source code in toolboxv2/utils/clis/db_cli_manager.py

def status_all(self, instance_id: str | None = None, silent=False):
    """Show status of all instances"""
    instances = self.get_instances(instance_id)

    if not silent:

        # Table header
        columns = [
            ("INSTANCE ID", 18),
            ("STATUS", 12),
            ("PID", 8),
            ("VERSION", 12),
            ("PORT", 6),
            ("HOST", 15)
        ]
        widths = [w for _, w in columns]

        print("🖥️ Cluster Status\n")
        print_table_header(columns, widths)

    services_online = 0
    server_list = []

    for instance in instances:
        pid, version = instance.read_state()
        is_running = instance.is_running()

        if is_running:
            server_list.append(f"http://{instance.host}:{instance.port}")
            services_online += 1

        if not silent:
            status_str = " RUNNING" if is_running else " STOPPED"
            status_style = "green" if is_running else "red"

            print_table_row(
                [
                    instance.id,
                    status_str,
                    str(pid or 'N/A'),
                    version or 'N/A',
                    str(instance.port),
                    instance.host
                ],
                widths,
                ["white", status_style, "grey", "blue", "yellow", "cyan"]
            )

    if not silent:
        print()
        print_status(f"Services online: {services_online}/{len(instances)}", "info")

    return services_online, server_list

stop_all(instance_id=None)

Stop all instances

Source code in toolboxv2/utils/clis/db_cli_manager.py

def stop_all(self, instance_id: str | None = None):
    """Stop all instances"""
    instances = self.get_instances(instance_id)

    if len(instances) > 1:
        print_box_header("Stopping Multiple Instances", "⏹️")
        print_box_content(f"Total instances: {len(instances)}", "info")
        print_box_footer()

    for instance in instances:
        instance.stop()
        if len(instances) > 1:
            print()

update_all_rolling(new_executable_path, new_version, instance_id=None)

Performs a zero-downtime rolling update of the cluster.

Source code in toolboxv2/utils/clis/db_cli_manager.py

def update_all_rolling(self, new_executable_path: Path, new_version: str, instance_id: str | None = None):
    """Performs a zero-downtime rolling update of the cluster."""
    instances_to_update = self.get_instances(instance_id)

    print_box_header(f"Rolling Update to Version {new_version}", "🔄")
    print_box_content(f"Instances to update: {len(instances_to_update)}", "info")
    print_box_content(f"Executable: {new_executable_path}", "info")
    print_box_footer()

    for i, instance in enumerate(instances_to_update):
        print_separator("═")
        print(f"  [{i + 1}/{len(instances_to_update)}] Updating instance '{instance.id}'")
        print_separator("═")
        print()

        if not instance.stop():
            print_status(f"CRITICAL: Failed to stop old instance '{instance.id}'. Aborting", "error")
            return

        if not instance.start(new_executable_path, new_version):
            print_status(f"CRITICAL: Failed to start new version for '{instance.id}'", "error")
            print_status("The cluster might be in a partially updated state", "warning")
            return

        # Health check with retries
        print()
        with Spinner(f"Waiting for '{instance.id}' to become healthy", symbols="t") as s:
            for attempt in range(5):
                s.message = f"Waiting for '{instance.id}' to become healthy (attempt {attempt + 1}/5)"
                time.sleep(2)
                health = instance.get_health()
                if health.get('status') == 'OK':
                    print()
                    print_status(f"Instance '{instance.id}' is healthy with new version", "success")
                    break
            else:
                print()
                print_status(f"Instance '{instance.id}' did not become healthy. Update halted", "error")
                return
        print()

    print_separator("═")
    print_status("Rolling Update Complete!", "success")
    print_separator("═")

DBInstanceManager

Manages a single r_blob_db instance.

Source code in toolboxv2/utils/clis/db_cli_manager.py

class DBInstanceManager:
    """Manages a single r_blob_db instance."""

    def __init__(self, instance_id: str, config: dict):
        self.id = instance_id
        self.port = config['port']
        self.host = config.get('host', '127.0.0.1')
        self.data_dir = Path(config['data_dir'])
        self.state_file = self.data_dir / "instance_state.json"
        self.log_file = self.data_dir / "instance.log"

    def read_state(self) -> tuple[int | None, str | None]:
        """Reads the PID and version from the instance's state file."""
        if not self.state_file.exists():
            return None, None
        try:
            with open(self.state_file) as f:
                state = json.load(f)
            return state.get('pid'), state.get('version')
        except (json.JSONDecodeError, ValueError, FileNotFoundError):
            return None, None

    def write_state(self, pid: int | None, version: str | None):
        """Writes the PID and version to the state file."""
        self.data_dir.mkdir(parents=True, exist_ok=True)
        state = {'pid': pid, 'version': version}
        with open(self.state_file, 'w') as f:
            json.dump(state, f, indent=4)

    def is_running(self) -> bool:
        """Checks if the process associated with this instance is running."""
        pid, _ = self.read_state()
        return psutil.pid_exists(pid) if pid else False

    def start(self, executable_path: Path, version: str) -> bool:
        """Starts the instance process and detaches, redirecting output to a log file."""
        if self.is_running():
            print_status(f"Instance '{self.id}' is already running", "warning")
            return True

        print_box_header(f"Starting Instance: {self.id}", "🚀")
        print_box_content(f"Port: {self.port}", "info")
        print_box_content(f"Data Directory: {str(self.data_dir)[:15]}...{str(self.data_dir)[-15:]}", "info")
        print_box_footer()

        self.data_dir.mkdir(parents=True, exist_ok=True)
        log_handle = open(self.log_file, 'a')

        env = os.environ.copy()
        env["R_BLOB_DB_CLEAN"] = os.getenv("R_BLOB_DB_CLEAN", "false")
        env["R_BLOB_DB_PORT"] = str(self.port)
        env["R_BLOB_DB_DATA_DIR"] = str(self.data_dir.resolve())
        env["RUST_LOG"] = "info,tower_http=debug"

        try:
            if executable_path is None:
                raise ValueError(f"Executable not found. Build it first.")

            with Spinner(f"Launching process for '{self.id}'", symbols="d"):
                process = subprocess.Popen(
                    [str(executable_path.resolve())],
                    env=env,
                    stdout=log_handle,
                    stderr=log_handle,
                    creationflags=subprocess.DETACHED_PROCESS if platform.system() == "Windows" else 0
                )
                time.sleep(1.5)

            if process.poll() is not None:
                print_status(f"Instance '{self.id}' failed to start. Check logs:", "error")
                print(f"    {Style.GREY(str(self.log_file))}")
                return False

            self.write_state(process.pid, version)
            print_status(f"Instance '{self.id}' started successfully (PID: {process.pid})", "success")
            print_status(f"Logging to: {self.log_file}", "info")
            return True

        except Exception as e:
            print_status(f"Failed to launch instance '{self.id}': {e}", "error")
            log_handle.close()
            return False

    def stop(self, timeout: int = 10) -> bool:
        """Stops the instance process gracefully."""
        if not self.is_running():
            print_status(f"Instance '{self.id}' is not running", "warning")
            self.write_state(None, None)
            return True

        pid, _ = self.read_state()

        print_box_header(f"Stopping Instance: {self.id}", "⏹️")
        print_box_content(f"PID: {pid}", "info")
        print_box_content(f"Timeout: {timeout}s", "info")
        print_box_footer()

        with Spinner(f"Stopping '{self.id}' (PID: {pid})", symbols="+", time_in_s=timeout, count_down=True) as s:
            try:
                proc = psutil.Process(pid)
                proc.terminate()
                proc.wait(timeout)
            except psutil.TimeoutExpired:
                s.message = f"Force killing '{self.id}'"
                proc.kill()
            except psutil.NoSuchProcess:
                pass
            except Exception as e:
                print_status(f"Failed to stop instance '{self.id}': {e}", "error")
                return False

        self.write_state(None, None)
        print_status(f"Instance '{self.id}' stopped", "success")
        return True

    def get_health(self) -> dict:
        """Performs a health check on the running instance."""
        if not self.is_running():
            return {'id': self.id, 'status': 'STOPPED', 'error': 'Process not running'}

        pid, version = self.read_state()
        health_url = f"http://{self.host}:{self.port}/health"
        start_time = time.monotonic()

        try:
            response = requests.get(health_url, timeout=2)
            latency_ms = (time.monotonic() - start_time) * 1000
            response.raise_for_status()
            health_data = response.json()
            health_data.update({
                'id': self.id,
                'pid': pid,
                'latency_ms': round(latency_ms),
                'server_version': health_data.pop('version', 'unknown'),
                'manager_known_version': version
            })
            return health_data
        except requests.exceptions.RequestException as e:
            return {'id': self.id, 'status': 'UNREACHABLE', 'pid': pid, 'error': str(e)}
        except Exception as e:
            return {'id': self.id, 'status': 'ERROR', 'pid': pid, 'error': f'Failed to parse health response: {e}'}

    def get_blob_list(self) -> List[Dict[str, Any]]:
        """Get list of all blobs from this instance"""
        if not self.is_running():
            return []

        # Try API endpoint first
        try:
            response = requests.get(f"http://{self.host}:{self.port}/blobs", timeout=5)
            response.raise_for_status()
            return response.json().get('blobs', [])
        except requests.exceptions.HTTPError as e:
            if e.response.status_code == 404:
                # Endpoint doesn't exist, fallback to scanning data directory
                print_status("API endpoint /blobs not available, scanning data directory...", "warning")
                return self._scan_data_directory()
            return []
        except Exception as e:
            print_status(f"Error fetching blob list: {e}", "warning")
            # Fallback to scanning data directory
            return self._scan_data_directory()

    def _scan_data_directory(self) -> List[Dict[str, Any]]:
        """Scan the data directory to find blobs (fallback method)"""
        blobs = []

        if not self.data_dir.exists():
            return []

        try:
            # Look for blob files in the data directory
            # Adjust patterns based on your Rust server's storage structure
            for item in self.data_dir.rglob('*'):
                if item.is_file() and not item.name.startswith('.'):
                    # Skip metadata files
                    if item.name in ['instance_state.json', 'instance.log']:
                        continue

                    # Get file stats
                    stat = item.stat()
                    blob_info = {
                        'id': item.stem if item.suffix == '.blob' else item.name,
                        'size': stat.st_size,
                        'created_at': time.strftime('%Y-%m-%d %H:%M:%S',
                                                    time.localtime(stat.st_ctime)),
                        'modified_at': time.strftime('%Y-%m-%d %H:%M:%S',
                                                     time.localtime(stat.st_mtime)),
                        'path': str(item.relative_to(self.data_dir))
                    }
                    blobs.append(blob_info)

            return sorted(blobs, key=lambda x: x['created_at'], reverse=True)

        except Exception as e:
            print_status(f"Error scanning data directory: {e}", "error")
            return []

    def get_blob_data(self, blob_id: str) -> Optional[bytes]:
        """Get data from a specific blob"""
        if not self.is_running():
            # Try to read from disk if server is not running
            return self._read_blob_from_disk(blob_id)

        try:
            response = requests.get(f"http://{self.host}:{self.port}/blob/{blob_id}", timeout=5)
            response.raise_for_status()
            return response.content
        except Exception as e:
            print_status(f"Error fetching blob via API: {e}, trying disk...", "warning")
            return self._read_blob_from_disk(blob_id)

    def _read_blob_from_disk(self, blob_id: str) -> Optional[bytes]:
        """Read blob directly from disk (fallback method)"""
        try:
            # Try different possible file patterns
            possible_paths = [
                self.data_dir / blob_id,
                self.data_dir / f"{blob_id}.blob",
                self.data_dir / "blobs" / blob_id,
                self.data_dir / "blobs" / f"{blob_id}.blob",
            ]

            for path in possible_paths:
                if path.exists() and path.is_file():
                    with open(path, 'rb') as f:
                        return f.read()

            return None
        except Exception as e:
            print_status(f"Error reading blob from disk: {e}", "error")
            return None

    def delete_blob(self, blob_id: str) -> bool:
        """Delete a specific blob"""
        if not self.is_running():
            print_status("Instance is not running, cannot delete blob via API", "error")
            return False

        try:
            response = requests.delete(f"http://{self.host}:{self.port}/blob/{blob_id}", timeout=5)
            response.raise_for_status()
            return True
        except Exception as e:
            print_status(f"Error deleting blob: {e}", "error")
            return False

    def get_stats(self) -> Dict[str, Any]:
        """Get detailed statistics from this instance"""
        if not self.is_running():
            # Return local stats if server not running
            return self._get_local_stats()

        try:
            # Try stats endpoint
            response = requests.get(f"http://{self.host}:{self.port}/stats", timeout=5)
            response.raise_for_status()
            return response.json()
        except requests.exceptions.HTTPError as e:
            if e.response.status_code == 404:
                # Stats endpoint doesn't exist, use health endpoint
                health = self.get_health()
                if health.get('status') == 'OK':
                    return {
                        'blob_count': health.get('blobs_managed', 0),
                        'total_size': 'N/A',
                        'status': 'OK'
                    }
            return self._get_local_stats()
        except Exception:
            return self._get_local_stats()

    def _get_local_stats(self) -> Dict[str, Any]:
        """Calculate statistics from local data directory"""
        try:
            if not self.data_dir.exists():
                return {'blob_count': 0, 'total_size': 0}

            total_size = 0
            blob_count = 0

            for item in self.data_dir.rglob('*'):
                if item.is_file() and not item.name.startswith('.'):
                    if item.name not in ['instance_state.json', 'instance.log']:
                        total_size += item.stat().st_size
                        blob_count += 1

            return {
                'blob_count': blob_count,
                'total_size': f"{total_size / 1024:.2f} KB" if total_size < 1024 * 1024
                else f"{total_size / (1024 * 1024):.2f} MB",
                'total_size_bytes': total_size,
                'status': 'offline' if not self.is_running() else 'unknown'
            }
        except Exception as e:
            return {'blob_count': 0, 'total_size': 0, 'error': str(e)}

delete_blob(blob_id)

Delete a specific blob

Source code in toolboxv2/utils/clis/db_cli_manager.py

def delete_blob(self, blob_id: str) -> bool:
    """Delete a specific blob"""
    if not self.is_running():
        print_status("Instance is not running, cannot delete blob via API", "error")
        return False

    try:
        response = requests.delete(f"http://{self.host}:{self.port}/blob/{blob_id}", timeout=5)
        response.raise_for_status()
        return True
    except Exception as e:
        print_status(f"Error deleting blob: {e}", "error")
        return False

get_blob_data(blob_id)

Get data from a specific blob

Source code in toolboxv2/utils/clis/db_cli_manager.py

def get_blob_data(self, blob_id: str) -> Optional[bytes]:
    """Get data from a specific blob"""
    if not self.is_running():
        # Try to read from disk if server is not running
        return self._read_blob_from_disk(blob_id)

    try:
        response = requests.get(f"http://{self.host}:{self.port}/blob/{blob_id}", timeout=5)
        response.raise_for_status()
        return response.content
    except Exception as e:
        print_status(f"Error fetching blob via API: {e}, trying disk...", "warning")
        return self._read_blob_from_disk(blob_id)

get_blob_list()

Get list of all blobs from this instance

Source code in toolboxv2/utils/clis/db_cli_manager.py

def get_blob_list(self) -> List[Dict[str, Any]]:
    """Get list of all blobs from this instance"""
    if not self.is_running():
        return []

    # Try API endpoint first
    try:
        response = requests.get(f"http://{self.host}:{self.port}/blobs", timeout=5)
        response.raise_for_status()
        return response.json().get('blobs', [])
    except requests.exceptions.HTTPError as e:
        if e.response.status_code == 404:
            # Endpoint doesn't exist, fallback to scanning data directory
            print_status("API endpoint /blobs not available, scanning data directory...", "warning")
            return self._scan_data_directory()
        return []
    except Exception as e:
        print_status(f"Error fetching blob list: {e}", "warning")
        # Fallback to scanning data directory
        return self._scan_data_directory()

get_health()

Performs a health check on the running instance.

Source code in toolboxv2/utils/clis/db_cli_manager.py

def get_health(self) -> dict:
    """Performs a health check on the running instance."""
    if not self.is_running():
        return {'id': self.id, 'status': 'STOPPED', 'error': 'Process not running'}

    pid, version = self.read_state()
    health_url = f"http://{self.host}:{self.port}/health"
    start_time = time.monotonic()

    try:
        response = requests.get(health_url, timeout=2)
        latency_ms = (time.monotonic() - start_time) * 1000
        response.raise_for_status()
        health_data = response.json()
        health_data.update({
            'id': self.id,
            'pid': pid,
            'latency_ms': round(latency_ms),
            'server_version': health_data.pop('version', 'unknown'),
            'manager_known_version': version
        })
        return health_data
    except requests.exceptions.RequestException as e:
        return {'id': self.id, 'status': 'UNREACHABLE', 'pid': pid, 'error': str(e)}
    except Exception as e:
        return {'id': self.id, 'status': 'ERROR', 'pid': pid, 'error': f'Failed to parse health response: {e}'}

get_stats()

Get detailed statistics from this instance

Source code in toolboxv2/utils/clis/db_cli_manager.py

def get_stats(self) -> Dict[str, Any]:
    """Get detailed statistics from this instance"""
    if not self.is_running():
        # Return local stats if server not running
        return self._get_local_stats()

    try:
        # Try stats endpoint
        response = requests.get(f"http://{self.host}:{self.port}/stats", timeout=5)
        response.raise_for_status()
        return response.json()
    except requests.exceptions.HTTPError as e:
        if e.response.status_code == 404:
            # Stats endpoint doesn't exist, use health endpoint
            health = self.get_health()
            if health.get('status') == 'OK':
                return {
                    'blob_count': health.get('blobs_managed', 0),
                    'total_size': 'N/A',
                    'status': 'OK'
                }
        return self._get_local_stats()
    except Exception:
        return self._get_local_stats()

is_running()

Checks if the process associated with this instance is running.

Source code in toolboxv2/utils/clis/db_cli_manager.py

def is_running(self) -> bool:
    """Checks if the process associated with this instance is running."""
    pid, _ = self.read_state()
    return psutil.pid_exists(pid) if pid else False

read_state()

Reads the PID and version from the instance's state file.

Source code in toolboxv2/utils/clis/db_cli_manager.py

def read_state(self) -> tuple[int | None, str | None]:
    """Reads the PID and version from the instance's state file."""
    if not self.state_file.exists():
        return None, None
    try:
        with open(self.state_file) as f:
            state = json.load(f)
        return state.get('pid'), state.get('version')
    except (json.JSONDecodeError, ValueError, FileNotFoundError):
        return None, None

start(executable_path, version)

Starts the instance process and detaches, redirecting output to a log file.

Source code in toolboxv2/utils/clis/db_cli_manager.py

def start(self, executable_path: Path, version: str) -> bool:
    """Starts the instance process and detaches, redirecting output to a log file."""
    if self.is_running():
        print_status(f"Instance '{self.id}' is already running", "warning")
        return True

    print_box_header(f"Starting Instance: {self.id}", "🚀")
    print_box_content(f"Port: {self.port}", "info")
    print_box_content(f"Data Directory: {str(self.data_dir)[:15]}...{str(self.data_dir)[-15:]}", "info")
    print_box_footer()

    self.data_dir.mkdir(parents=True, exist_ok=True)
    log_handle = open(self.log_file, 'a')

    env = os.environ.copy()
    env["R_BLOB_DB_CLEAN"] = os.getenv("R_BLOB_DB_CLEAN", "false")
    env["R_BLOB_DB_PORT"] = str(self.port)
    env["R_BLOB_DB_DATA_DIR"] = str(self.data_dir.resolve())
    env["RUST_LOG"] = "info,tower_http=debug"

    try:
        if executable_path is None:
            raise ValueError(f"Executable not found. Build it first.")

        with Spinner(f"Launching process for '{self.id}'", symbols="d"):
            process = subprocess.Popen(
                [str(executable_path.resolve())],
                env=env,
                stdout=log_handle,
                stderr=log_handle,
                creationflags=subprocess.DETACHED_PROCESS if platform.system() == "Windows" else 0
            )
            time.sleep(1.5)

        if process.poll() is not None:
            print_status(f"Instance '{self.id}' failed to start. Check logs:", "error")
            print(f"    {Style.GREY(str(self.log_file))}")
            return False

        self.write_state(process.pid, version)
        print_status(f"Instance '{self.id}' started successfully (PID: {process.pid})", "success")
        print_status(f"Logging to: {self.log_file}", "info")
        return True

    except Exception as e:
        print_status(f"Failed to launch instance '{self.id}': {e}", "error")
        log_handle.close()
        return False

stop(timeout=10)

Stops the instance process gracefully.

Source code in toolboxv2/utils/clis/db_cli_manager.py

def stop(self, timeout: int = 10) -> bool:
    """Stops the instance process gracefully."""
    if not self.is_running():
        print_status(f"Instance '{self.id}' is not running", "warning")
        self.write_state(None, None)
        return True

    pid, _ = self.read_state()

    print_box_header(f"Stopping Instance: {self.id}", "⏹️")
    print_box_content(f"PID: {pid}", "info")
    print_box_content(f"Timeout: {timeout}s", "info")
    print_box_footer()

    with Spinner(f"Stopping '{self.id}' (PID: {pid})", symbols="+", time_in_s=timeout, count_down=True) as s:
        try:
            proc = psutil.Process(pid)
            proc.terminate()
            proc.wait(timeout)
        except psutil.TimeoutExpired:
            s.message = f"Force killing '{self.id}'"
            proc.kill()
        except psutil.NoSuchProcess:
            pass
        except Exception as e:
            print_status(f"Failed to stop instance '{self.id}': {e}", "error")
            return False

    self.write_state(None, None)
    print_status(f"Instance '{self.id}' stopped", "success")
    return True

write_state(pid, version)

Writes the PID and version to the state file.

Source code in toolboxv2/utils/clis/db_cli_manager.py

def write_state(self, pid: int | None, version: str | None):
    """Writes the PID and version to the state file."""
    self.data_dir.mkdir(parents=True, exist_ok=True)
    state = {'pid': pid, 'version': version}
    with open(self.state_file, 'w') as f:
        json.dump(state, f, indent=4)

DataDiscovery

Interactive data discovery and manipulation interface

Source code in toolboxv2/utils/clis/db_cli_manager.py

class DataDiscovery:
    """Interactive data discovery and manipulation interface"""

    def __init__(self, manager: ClusterManager):
        self.manager = manager
        self.selected_instance: Optional[DBInstanceManager] = None
        self.current_view = "instances"  # instances, blobs, blob_detail
        self.selected_index = 0
        self.blob_list = []
        self.current_blob = None

    async def run(self):
        """Run interactive discovery session"""
        print('\033[2J\033[H')  # Clear screen

        print_box_header("Interactive Data Discovery & Manipulation", "🔍")
        print_box_content("Navigate with ↑↓ or w/s, Enter to select, q to quit", "info")
        print_box_footer()

        while True:
            if self.current_view == "instances":
                action = self.show_instances()
            elif self.current_view == "blobs":
                action = self.show_blobs()
            elif self.current_view == "blob_detail":
                action = self.show_blob_detail()

            if action == "quit":
                break
            elif action == "back":
                self.go_back()

    def show_instances(self):
        """Show instance selection menu with keyboard navigation"""
        import sys

        def get_key():
            """Get single keypress (cross-platform)"""
            if platform.system() == "Windows":
                import msvcrt
                key = msvcrt.getch()
                if key == b'\xe0':  # Arrow key prefix on Windows
                    key = msvcrt.getch()
                    if key == b'H':
                        return 'up'
                    elif key == b'P':
                        return 'down'
                elif key == b'\r':
                    return 'enter'
                elif key in (b'q', b'Q'):
                    return 'quit'
                elif key in (b'w', b'W'):
                    return 'up'
                elif key in (b's', b'S'):
                    return 'down'
                return None
            else:
                import tty
                import termios
                fd = sys.stdin.fileno()
                old_settings = termios.tcgetattr(fd)
                try:
                    tty.setraw(sys.stdin.fileno())
                    ch = sys.stdin.read(1)
                    if ch == '\x1b':  # ESC sequence
                        next_chars = sys.stdin.read(2)
                        if next_chars == '[A':
                            return 'up'
                        elif next_chars == '[B':
                            return 'down'
                    elif ch in ('\r', '\n'):
                        return 'enter'
                    elif ch in ('q', 'Q', '\x03'):  # q or Ctrl+C
                        return 'quit'
                    elif ch in ('w', 'W'):
                        return 'up'
                    elif ch in ('s', 'S'):
                        return 'down'
                    return None
                finally:
                    termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)

        while True:
            print('\033[2J\033[H')  # Clear screen

            print_box_header("Select Database Instance", "🗄️")
            print()

            instances = self.manager.get_instances()

            for i, instance in enumerate(instances):
                is_selected = i == self.selected_index
                is_running = instance.is_running()

                status_icon = "✅" if is_running else "❌"
                arrow = "▶" if is_selected else " "

                if is_selected:
                    print(f"  {arrow} \033[1;96m{status_icon} {instance.id:<20} Port: {instance.port:<6}\033[0m")
                else:
                    print(f"  {arrow} {status_icon} {instance.id:<20} Port: {instance.port:<6}")

            print()
            print_box_footer()
            print_status("↑↓/w/s: Navigate | Enter: Select | q: Quit", "info")

            # Get single keypress
            key = get_key()

            if key == 'quit':
                return "quit"
            elif key == 'up':
                self.selected_index = max(0, self.selected_index - 1)
            elif key == 'down':
                self.selected_index = min(len(instances) - 1, self.selected_index + 1)
            elif key == 'enter':
                self.selected_instance = instances[self.selected_index]
                if not self.selected_instance.is_running():
                    print()
                    print_status("Instance is not running! Please start it first.", "error")
                    time.sleep(2)
                else:
                    self.current_view = "blobs"
                    self.selected_index = 0
                    self.load_blobs()
                    return "continue"

    def load_blobs(self):
        """Load blob list from selected instance"""
        if not self.selected_instance:
            return

        print_status("Loading blobs...", "progress")
        self.blob_list = self.selected_instance.get_blob_list()

    def show_blobs(self):
        """Show blob list with keyboard navigation"""
        import sys

        def get_key():
            """Get single keypress (cross-platform)"""
            if platform.system() == "Windows":
                import msvcrt
                key = msvcrt.getch()
                if key == b'\xe0':  # Arrow key prefix on Windows
                    key = msvcrt.getch()
                    if key == b'H':
                        return 'up'
                    elif key == b'P':
                        return 'down'
                elif key == b'\r':
                    return 'enter'
                elif key in (b'q', b'Q'):
                    return 'quit'
                elif key in (b'w', b'W'):
                    return 'up'
                elif key in (b's', b'S'):
                    return 'down'
                elif key in (b'd', b'D'):
                    return 'delete'
                elif key in (b'r', b'R'):
                    return 'refresh'
                elif key in (b'b', b'B'):
                    return 'back'
                return None
            else:
                import tty
                import termios
                fd = sys.stdin.fileno()
                old_settings = termios.tcgetattr(fd)
                try:
                    tty.setraw(sys.stdin.fileno())
                    ch = sys.stdin.read(1)
                    if ch == '\x1b':  # ESC sequence
                        next_chars = sys.stdin.read(2)
                        if next_chars == '[A':
                            return 'up'
                        elif next_chars == '[B':
                            return 'down'
                    elif ch in ('\r', '\n'):
                        return 'enter'
                    elif ch in ('q', 'Q', '\x03'):  # q or Ctrl+C
                        return 'quit'
                    elif ch in ('w', 'W'):
                        return 'up'
                    elif ch in ('s', 'S'):
                        return 'down'
                    elif ch in ('d', 'D'):
                        return 'delete'
                    elif ch in ('r', 'R'):
                        return 'refresh'
                    elif ch in ('b', 'B'):
                        return 'back'
                    return None
                finally:
                    termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)

        while True:
            print('\033[2J\033[H')  # Clear screen

            print_box_header(f"Blobs in {self.selected_instance.id}", "📦")
            print()

            if not self.blob_list:
                print_box_content("No blobs found in this instance", "warning")
                print_box_footer()
                print_status("Press 'b' to go back or 'r' to refresh...", "info")

                key = get_key()
                if key in ('back', 'quit'):
                    return "back"
                elif key == 'refresh':
                    self.load_blobs()
                continue

            # Show stats
            stats = self.selected_instance.get_stats()
            if stats:
                print(f"  Total Blobs: {len(self.blob_list)}")
                if 'total_size' in stats:
                    print(f"  Total Size: {stats.get('total_size', 'N/A')}")
                print()

            # Show blob list
            print_separator()

            # Calculate visible range (show 15 items at a time)
            visible_count = 15
            start_idx = max(0, self.selected_index - visible_count // 2)
            end_idx = min(len(self.blob_list), start_idx + visible_count)

            # Adjust start if we're near the end
            if end_idx - start_idx < visible_count:
                start_idx = max(0, end_idx - visible_count)

            for i in range(start_idx, end_idx):
                blob = self.blob_list[i]
                is_selected = i == self.selected_index
                arrow = "▶" if is_selected else " "

                blob_id = blob.get('id', 'N/A')
                blob_id_display = (blob_id[:37] + '...') if len(blob_id) > 40 else blob_id
                blob_size = blob.get('size', 0)
                size_str = self.format_size(blob_size)

                if is_selected:
                    print(f"  {arrow} \033[1;96m{blob_id_display:<42} {size_str:>10}\033[0m")
                else:
                    print(f"  {arrow} {blob_id_display:<42} {size_str:>10}")

            if len(self.blob_list) > visible_count:
                print(f"\n  Showing {start_idx + 1}-{end_idx} of {len(self.blob_list)}")

            print()
            print_box_footer()
            print_status("↑↓/w/s: Navigate | Enter: View | d: Delete | r: Refresh | b: Back | q: Quit", "info")

            # Get user input
            key = get_key()

            if key == 'quit':
                return "quit"
            elif key == 'back':
                return "back"
            elif key == 'refresh':
                print()
                print_status("Refreshing blob list...", "progress")
                self.load_blobs()
                self.selected_index = min(self.selected_index, len(self.blob_list) - 1)
                time.sleep(0.5)
            elif key == 'up':
                self.selected_index = max(0, self.selected_index - 1)
            elif key == 'down':
                self.selected_index = min(len(self.blob_list) - 1, self.selected_index + 1)
            elif key == 'delete':
                if 0 <= self.selected_index < len(self.blob_list):
                    if self.confirm_delete_blob():
                        self.delete_current_blob()
                        # Adjust selected index if needed
                        if self.selected_index >= len(self.blob_list):
                            self.selected_index = max(0, len(self.blob_list) - 1)
            elif key == 'enter':
                if 0 <= self.selected_index < len(self.blob_list):
                    self.current_blob = self.blob_list[self.selected_index]
                    self.current_view = "blob_detail"
                    return "continue"

    def show_blob_detail(self):
        """Show detailed view of a blob with keyboard navigation"""
        import sys

        def get_key():
            """Get single keypress (cross-platform)"""
            if platform.system() == "Windows":
                import msvcrt
                key = msvcrt.getch()
                if key == b'\xe0':  # Arrow key prefix on Windows
                    key = msvcrt.getch()
                elif key in (b'q', b'Q'):
                    return 'quit'
                elif key in (b'e', b'E'):
                    return 'export'
                elif key in (b'd', b'D'):
                    return 'delete'
                elif key in (b'b', b'B'):
                    return 'back'
                return None
            else:
                import tty
                import termios
                fd = sys.stdin.fileno()
                old_settings = termios.tcgetattr(fd)
                try:
                    tty.setraw(sys.stdin.fileno())
                    ch = sys.stdin.read(1)
                    if ch == '\x1b':  # ESC sequence
                        sys.stdin.read(2)  # Consume arrow key codes
                    elif ch in ('q', 'Q', '\x03'):  # q or Ctrl+C
                        return 'quit'
                    elif ch in ('e', 'E'):
                        return 'export'
                    elif ch in ('d', 'D'):
                        return 'delete'
                    elif ch in ('b', 'B'):
                        return 'back'
                    return None
                finally:
                    termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)

        while True:
            print('\033[2J\033[H')  # Clear screen

            blob_id = self.current_blob.get('id', 'N/A')

            print_box_header(f"Blob Details", "📄")
            print()
            print(f"  ID: {blob_id}")
            print(f"  Size: {self.format_size(self.current_blob.get('size', 0))}")

            if 'created_at' in self.current_blob:
                print(f"  Created: {self.current_blob.get('created_at', 'N/A')}")
            if 'modified_at' in self.current_blob:
                print(f"  Modified: {self.current_blob.get('modified_at', 'N/A')}")
            if 'path' in self.current_blob:
                print(f"  Path: {self.current_blob.get('path', 'N/A')}")

            print()

            # Try to load and preview data
            print_status("Loading blob data...", "progress")
            data = self.selected_instance.get_blob_data(blob_id)

            if data:
                print()
                print_separator()
                print("  Data Preview (first 500 bytes):")
                print_separator()

                try:
                    # Try to decode as text
                    text_preview = data[:500].decode('utf-8', errors='ignore')
                    # Replace control characters except newlines
                    text_preview = ''.join(
                        char if char == '\n' or (32 <= ord(char) < 127) else '.'
                        for char in text_preview
                    )
                    print(f"\n{text_preview}\n")
                except:
                    # Show hex dump
                    hex_preview = data[:200].hex()
                    # Format as rows of 32 hex chars
                    for i in range(0, len(hex_preview), 32):
                        print(f"  {hex_preview[i:i + 32]}")
                    print()

                print_separator()
            else:
                print()
                print_status("Could not load blob data", "error")

            print()
            print_box_footer()
            print_status("e: Export | d: Delete | b: Back | q: Quit", "info")

            # Get user input
            key = get_key()

            if key == 'quit':
                return "quit"
            elif key == 'back':
                return "back"
            elif key == 'export':
                if data:
                    # Temporarily restore terminal for input
                    if platform.system() != "Windows":
                        import tty
                        import termios
                        fd = sys.stdin.fileno()
                        old_settings = termios.tcgetattr(fd)
                        termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)

                    self.export_current_blob(data)

                    # Wait for keypress to continue
                    print()
                    print_status("Press any key to continue...", "info")
                    get_key()
                else:
                    print()
                    print_status("No data to export", "error")
                    time.sleep(1)
            elif key == 'delete':
                # Temporarily restore terminal for confirmation input
                if platform.system() != "Windows":
                    import tty
                    import termios
                    fd = sys.stdin.fileno()
                    old_settings = termios.tcgetattr(fd)
                    termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)

                if self.confirm_delete_blob():
                    self.delete_current_blob()
                    return "back"

    def confirm_delete_blob(self) -> bool:
        """Confirm blob deletion with user"""
        if self.current_view == "blob_detail":
            blob = self.current_blob
        elif self.current_view == "blobs":
            if 0 <= self.selected_index < len(self.blob_list):
                blob = self.blob_list[self.selected_index]
            else:
                return False
        else:
            return False

        blob_id = blob.get('id', 'N/A')
        blob_id_short = (blob_id[:20] + '...') if len(blob_id) > 23 else blob_id

        print()
        print_status(f"Really delete blob {blob_id_short}?", "warning")
        print("  Type 'yes' to confirm: ", end='', flush=True)

        # Restore normal input temporarily
        if platform.system() != "Windows":
            import tty
            import termios
            import sys
            fd = sys.stdin.fileno()
            old_settings = termios.tcgetattr(fd)
            termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)

        confirm = input().strip().lower()
        return confirm == 'yes'

    def delete_current_blob(self):
        """Delete the currently selected blob"""
        if self.current_view == "blob_detail":
            blob = self.current_blob
        elif self.current_view == "blobs":
            if 0 <= self.selected_index < len(self.blob_list):
                blob = self.blob_list[self.selected_index]
            else:
                return
        else:
            return

        blob_id = blob.get('id', 'N/A')

        print()
        print_status(f"Deleting blob...", "progress")

        if self.selected_instance.delete_blob(blob_id):
            print_status("Blob deleted successfully", "success")
            self.load_blobs()  # Refresh list
        else:
            print_status("Failed to delete blob", "error")

        time.sleep(1)

    def export_current_blob(self, data: bytes):
        """Export blob data to file"""
        print()
        print("  Enter filename to export to: ", end='', flush=True)

        filename = input().strip()

        if filename:
            try:
                with open(filename, 'wb') as f:
                    f.write(data)
                print_status(f"Blob exported to {filename}", "success")
            except Exception as e:
                print_status(f"Export failed: {e}", "error")
        else:
            print_status("Export cancelled", "warning")

    def go_back(self):
        """Navigate back in the view hierarchy"""
        if self.current_view == "blob_detail":
            self.current_view = "blobs"
            self.current_blob = None
        elif self.current_view == "blobs":
            self.current_view = "instances"
            self.selected_instance = None
            self.blob_list = []
            self.selected_index = 0

    @staticmethod
    def format_size(size_bytes: int) -> str:
        """Format byte size to human readable"""
        for unit in ['B', 'KB', 'MB', 'GB']:
            if size_bytes < 1024.0:
                return f"{size_bytes:.1f}{unit}"
            size_bytes /= 1024.0
        return f"{size_bytes:.1f}TB"

confirm_delete_blob()

Confirm blob deletion with user

Source code in toolboxv2/utils/clis/db_cli_manager.py

def confirm_delete_blob(self) -> bool:
    """Confirm blob deletion with user"""
    if self.current_view == "blob_detail":
        blob = self.current_blob
    elif self.current_view == "blobs":
        if 0 <= self.selected_index < len(self.blob_list):
            blob = self.blob_list[self.selected_index]
        else:
            return False
    else:
        return False

    blob_id = blob.get('id', 'N/A')
    blob_id_short = (blob_id[:20] + '...') if len(blob_id) > 23 else blob_id

    print()
    print_status(f"Really delete blob {blob_id_short}?", "warning")
    print("  Type 'yes' to confirm: ", end='', flush=True)

    # Restore normal input temporarily
    if platform.system() != "Windows":
        import tty
        import termios
        import sys
        fd = sys.stdin.fileno()
        old_settings = termios.tcgetattr(fd)
        termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)

    confirm = input().strip().lower()
    return confirm == 'yes'

delete_current_blob()

Delete the currently selected blob

Source code in toolboxv2/utils/clis/db_cli_manager.py

def delete_current_blob(self):
    """Delete the currently selected blob"""
    if self.current_view == "blob_detail":
        blob = self.current_blob
    elif self.current_view == "blobs":
        if 0 <= self.selected_index < len(self.blob_list):
            blob = self.blob_list[self.selected_index]
        else:
            return
    else:
        return

    blob_id = blob.get('id', 'N/A')

    print()
    print_status(f"Deleting blob...", "progress")

    if self.selected_instance.delete_blob(blob_id):
        print_status("Blob deleted successfully", "success")
        self.load_blobs()  # Refresh list
    else:
        print_status("Failed to delete blob", "error")

    time.sleep(1)

export_current_blob(data)

Export blob data to file

Source code in toolboxv2/utils/clis/db_cli_manager.py

def export_current_blob(self, data: bytes):
    """Export blob data to file"""
    print()
    print("  Enter filename to export to: ", end='', flush=True)

    filename = input().strip()

    if filename:
        try:
            with open(filename, 'wb') as f:
                f.write(data)
            print_status(f"Blob exported to {filename}", "success")
        except Exception as e:
            print_status(f"Export failed: {e}", "error")
    else:
        print_status("Export cancelled", "warning")

format_size(size_bytes) staticmethod

Format byte size to human readable

Source code in toolboxv2/utils/clis/db_cli_manager.py

@staticmethod
def format_size(size_bytes: int) -> str:
    """Format byte size to human readable"""
    for unit in ['B', 'KB', 'MB', 'GB']:
        if size_bytes < 1024.0:
            return f"{size_bytes:.1f}{unit}"
        size_bytes /= 1024.0
    return f"{size_bytes:.1f}TB"

go_back()

Navigate back in the view hierarchy

Source code in toolboxv2/utils/clis/db_cli_manager.py

def go_back(self):
    """Navigate back in the view hierarchy"""
    if self.current_view == "blob_detail":
        self.current_view = "blobs"
        self.current_blob = None
    elif self.current_view == "blobs":
        self.current_view = "instances"
        self.selected_instance = None
        self.blob_list = []
        self.selected_index = 0

load_blobs()

Load blob list from selected instance

Source code in toolboxv2/utils/clis/db_cli_manager.py

def load_blobs(self):
    """Load blob list from selected instance"""
    if not self.selected_instance:
        return

    print_status("Loading blobs...", "progress")
    self.blob_list = self.selected_instance.get_blob_list()

run() async

Run interactive discovery session

Source code in toolboxv2/utils/clis/db_cli_manager.py

async def run(self):
    """Run interactive discovery session"""
    print('\033[2J\033[H')  # Clear screen

    print_box_header("Interactive Data Discovery & Manipulation", "🔍")
    print_box_content("Navigate with ↑↓ or w/s, Enter to select, q to quit", "info")
    print_box_footer()

    while True:
        if self.current_view == "instances":
            action = self.show_instances()
        elif self.current_view == "blobs":
            action = self.show_blobs()
        elif self.current_view == "blob_detail":
            action = self.show_blob_detail()

        if action == "quit":
            break
        elif action == "back":
            self.go_back()

show_blob_detail()

Show detailed view of a blob with keyboard navigation

Source code in toolboxv2/utils/clis/db_cli_manager.py

def show_blob_detail(self):
    """Show detailed view of a blob with keyboard navigation"""
    import sys

    def get_key():
        """Get single keypress (cross-platform)"""
        if platform.system() == "Windows":
            import msvcrt
            key = msvcrt.getch()
            if key == b'\xe0':  # Arrow key prefix on Windows
                key = msvcrt.getch()
            elif key in (b'q', b'Q'):
                return 'quit'
            elif key in (b'e', b'E'):
                return 'export'
            elif key in (b'd', b'D'):
                return 'delete'
            elif key in (b'b', b'B'):
                return 'back'
            return None
        else:
            import tty
            import termios
            fd = sys.stdin.fileno()
            old_settings = termios.tcgetattr(fd)
            try:
                tty.setraw(sys.stdin.fileno())
                ch = sys.stdin.read(1)
                if ch == '\x1b':  # ESC sequence
                    sys.stdin.read(2)  # Consume arrow key codes
                elif ch in ('q', 'Q', '\x03'):  # q or Ctrl+C
                    return 'quit'
                elif ch in ('e', 'E'):
                    return 'export'
                elif ch in ('d', 'D'):
                    return 'delete'
                elif ch in ('b', 'B'):
                    return 'back'
                return None
            finally:
                termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)

    while True:
        print('\033[2J\033[H')  # Clear screen

        blob_id = self.current_blob.get('id', 'N/A')

        print_box_header(f"Blob Details", "📄")
        print()
        print(f"  ID: {blob_id}")
        print(f"  Size: {self.format_size(self.current_blob.get('size', 0))}")

        if 'created_at' in self.current_blob:
            print(f"  Created: {self.current_blob.get('created_at', 'N/A')}")
        if 'modified_at' in self.current_blob:
            print(f"  Modified: {self.current_blob.get('modified_at', 'N/A')}")
        if 'path' in self.current_blob:
            print(f"  Path: {self.current_blob.get('path', 'N/A')}")

        print()

        # Try to load and preview data
        print_status("Loading blob data...", "progress")
        data = self.selected_instance.get_blob_data(blob_id)

        if data:
            print()
            print_separator()
            print("  Data Preview (first 500 bytes):")
            print_separator()

            try:
                # Try to decode as text
                text_preview = data[:500].decode('utf-8', errors='ignore')
                # Replace control characters except newlines
                text_preview = ''.join(
                    char if char == '\n' or (32 <= ord(char) < 127) else '.'
                    for char in text_preview
                )
                print(f"\n{text_preview}\n")
            except:
                # Show hex dump
                hex_preview = data[:200].hex()
                # Format as rows of 32 hex chars
                for i in range(0, len(hex_preview), 32):
                    print(f"  {hex_preview[i:i + 32]}")
                print()

            print_separator()
        else:
            print()
            print_status("Could not load blob data", "error")

        print()
        print_box_footer()
        print_status("e: Export | d: Delete | b: Back | q: Quit", "info")

        # Get user input
        key = get_key()

        if key == 'quit':
            return "quit"
        elif key == 'back':
            return "back"
        elif key == 'export':
            if data:
                # Temporarily restore terminal for input
                if platform.system() != "Windows":
                    import tty
                    import termios
                    fd = sys.stdin.fileno()
                    old_settings = termios.tcgetattr(fd)
                    termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)

                self.export_current_blob(data)

                # Wait for keypress to continue
                print()
                print_status("Press any key to continue...", "info")
                get_key()
            else:
                print()
                print_status("No data to export", "error")
                time.sleep(1)
        elif key == 'delete':
            # Temporarily restore terminal for confirmation input
            if platform.system() != "Windows":
                import tty
                import termios
                fd = sys.stdin.fileno()
                old_settings = termios.tcgetattr(fd)
                termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)

            if self.confirm_delete_blob():
                self.delete_current_blob()
                return "back"

show_blobs()

Show blob list with keyboard navigation

Source code in toolboxv2/utils/clis/db_cli_manager.py

def show_blobs(self):
    """Show blob list with keyboard navigation"""
    import sys

    def get_key():
        """Get single keypress (cross-platform)"""
        if platform.system() == "Windows":
            import msvcrt
            key = msvcrt.getch()
            if key == b'\xe0':  # Arrow key prefix on Windows
                key = msvcrt.getch()
                if key == b'H':
                    return 'up'
                elif key == b'P':
                    return 'down'
            elif key == b'\r':
                return 'enter'
            elif key in (b'q', b'Q'):
                return 'quit'
            elif key in (b'w', b'W'):
                return 'up'
            elif key in (b's', b'S'):
                return 'down'
            elif key in (b'd', b'D'):
                return 'delete'
            elif key in (b'r', b'R'):
                return 'refresh'
            elif key in (b'b', b'B'):
                return 'back'
            return None
        else:
            import tty
            import termios
            fd = sys.stdin.fileno()
            old_settings = termios.tcgetattr(fd)
            try:
                tty.setraw(sys.stdin.fileno())
                ch = sys.stdin.read(1)
                if ch == '\x1b':  # ESC sequence
                    next_chars = sys.stdin.read(2)
                    if next_chars == '[A':
                        return 'up'
                    elif next_chars == '[B':
                        return 'down'
                elif ch in ('\r', '\n'):
                    return 'enter'
                elif ch in ('q', 'Q', '\x03'):  # q or Ctrl+C
                    return 'quit'
                elif ch in ('w', 'W'):
                    return 'up'
                elif ch in ('s', 'S'):
                    return 'down'
                elif ch in ('d', 'D'):
                    return 'delete'
                elif ch in ('r', 'R'):
                    return 'refresh'
                elif ch in ('b', 'B'):
                    return 'back'
                return None
            finally:
                termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)

    while True:
        print('\033[2J\033[H')  # Clear screen

        print_box_header(f"Blobs in {self.selected_instance.id}", "📦")
        print()

        if not self.blob_list:
            print_box_content("No blobs found in this instance", "warning")
            print_box_footer()
            print_status("Press 'b' to go back or 'r' to refresh...", "info")

            key = get_key()
            if key in ('back', 'quit'):
                return "back"
            elif key == 'refresh':
                self.load_blobs()
            continue

        # Show stats
        stats = self.selected_instance.get_stats()
        if stats:
            print(f"  Total Blobs: {len(self.blob_list)}")
            if 'total_size' in stats:
                print(f"  Total Size: {stats.get('total_size', 'N/A')}")
            print()

        # Show blob list
        print_separator()

        # Calculate visible range (show 15 items at a time)
        visible_count = 15
        start_idx = max(0, self.selected_index - visible_count // 2)
        end_idx = min(len(self.blob_list), start_idx + visible_count)

        # Adjust start if we're near the end
        if end_idx - start_idx < visible_count:
            start_idx = max(0, end_idx - visible_count)

        for i in range(start_idx, end_idx):
            blob = self.blob_list[i]
            is_selected = i == self.selected_index
            arrow = "▶" if is_selected else " "

            blob_id = blob.get('id', 'N/A')
            blob_id_display = (blob_id[:37] + '...') if len(blob_id) > 40 else blob_id
            blob_size = blob.get('size', 0)
            size_str = self.format_size(blob_size)

            if is_selected:
                print(f"  {arrow} \033[1;96m{blob_id_display:<42} {size_str:>10}\033[0m")
            else:
                print(f"  {arrow} {blob_id_display:<42} {size_str:>10}")

        if len(self.blob_list) > visible_count:
            print(f"\n  Showing {start_idx + 1}-{end_idx} of {len(self.blob_list)}")

        print()
        print_box_footer()
        print_status("↑↓/w/s: Navigate | Enter: View | d: Delete | r: Refresh | b: Back | q: Quit", "info")

        # Get user input
        key = get_key()

        if key == 'quit':
            return "quit"
        elif key == 'back':
            return "back"
        elif key == 'refresh':
            print()
            print_status("Refreshing blob list...", "progress")
            self.load_blobs()
            self.selected_index = min(self.selected_index, len(self.blob_list) - 1)
            time.sleep(0.5)
        elif key == 'up':
            self.selected_index = max(0, self.selected_index - 1)
        elif key == 'down':
            self.selected_index = min(len(self.blob_list) - 1, self.selected_index + 1)
        elif key == 'delete':
            if 0 <= self.selected_index < len(self.blob_list):
                if self.confirm_delete_blob():
                    self.delete_current_blob()
                    # Adjust selected index if needed
                    if self.selected_index >= len(self.blob_list):
                        self.selected_index = max(0, len(self.blob_list) - 1)
        elif key == 'enter':
            if 0 <= self.selected_index < len(self.blob_list):
                self.current_blob = self.blob_list[self.selected_index]
                self.current_view = "blob_detail"
                return "continue"

show_instances()

Show instance selection menu with keyboard navigation

Source code in toolboxv2/utils/clis/db_cli_manager.py

def show_instances(self):
    """Show instance selection menu with keyboard navigation"""
    import sys

    def get_key():
        """Get single keypress (cross-platform)"""
        if platform.system() == "Windows":
            import msvcrt
            key = msvcrt.getch()
            if key == b'\xe0':  # Arrow key prefix on Windows
                key = msvcrt.getch()
                if key == b'H':
                    return 'up'
                elif key == b'P':
                    return 'down'
            elif key == b'\r':
                return 'enter'
            elif key in (b'q', b'Q'):
                return 'quit'
            elif key in (b'w', b'W'):
                return 'up'
            elif key in (b's', b'S'):
                return 'down'
            return None
        else:
            import tty
            import termios
            fd = sys.stdin.fileno()
            old_settings = termios.tcgetattr(fd)
            try:
                tty.setraw(sys.stdin.fileno())
                ch = sys.stdin.read(1)
                if ch == '\x1b':  # ESC sequence
                    next_chars = sys.stdin.read(2)
                    if next_chars == '[A':
                        return 'up'
                    elif next_chars == '[B':
                        return 'down'
                elif ch in ('\r', '\n'):
                    return 'enter'
                elif ch in ('q', 'Q', '\x03'):  # q or Ctrl+C
                    return 'quit'
                elif ch in ('w', 'W'):
                    return 'up'
                elif ch in ('s', 'S'):
                    return 'down'
                return None
            finally:
                termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)

    while True:
        print('\033[2J\033[H')  # Clear screen

        print_box_header("Select Database Instance", "🗄️")
        print()

        instances = self.manager.get_instances()

        for i, instance in enumerate(instances):
            is_selected = i == self.selected_index
            is_running = instance.is_running()

            status_icon = "✅" if is_running else "❌"
            arrow = "▶" if is_selected else " "

            if is_selected:
                print(f"  {arrow} \033[1;96m{status_icon} {instance.id:<20} Port: {instance.port:<6}\033[0m")
            else:
                print(f"  {arrow} {status_icon} {instance.id:<20} Port: {instance.port:<6}")

        print()
        print_box_footer()
        print_status("↑↓/w/s: Navigate | Enter: Select | q: Quit", "info")

        # Get single keypress
        key = get_key()

        if key == 'quit':
            return "quit"
        elif key == 'up':
            self.selected_index = max(0, self.selected_index - 1)
        elif key == 'down':
            self.selected_index = min(len(instances) - 1, self.selected_index + 1)
        elif key == 'enter':
            self.selected_instance = instances[self.selected_index]
            if not self.selected_instance.is_running():
                print()
                print_status("Instance is not running! Please start it first.", "error")
                time.sleep(2)
            else:
                self.current_view = "blobs"
                self.selected_index = 0
                self.load_blobs()
                return "continue"

cli_db_runner() async

The main entry point for the CLI application.

Source code in toolboxv2/utils/clis/db_cli_manager.py

async def cli_db_runner():
    """The main entry point for the CLI application."""

    parser = argparse.ArgumentParser(
        description="🗄️  r_blob_db Cluster Manager - Interactive Database Management",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        prog='tb db',
        epilog="""
╔════════════════════════════════════════════════════════════════════════════╗
║                           Command Examples                                 ║
╠════════════════════════════════════════════════════════════════════════════╣
║                                                                            ║
║  Build & Setup:                                                            ║
║    $ tb db build                     # Build Rust executable               ║
║    $ tb db clean                     # Clean build artifacts               ║
║                                                                            ║
║  Instance Management:                                                      ║
║    $ tb db start                     # Start all instances                 ║
║    $ tb db start --instance-id i1    # Start specific instance             ║
║    $ tb db stop                      # Stop all instances                  ║
║    $ tb db status                    # Show instance status                ║
║                                                                            ║
║  Health & Monitoring:                                                      ║
║    $ tb db health                    # Health check all instances          ║
║    $ tb db health --instance-id i1   # Check specific instance             ║
║                                                                            ║
║  Data Discovery:                                                           ║
║    $ tb db discover                  # Interactive data browser            ║
║                                                                            ║
║  Updates:                                                                  ║
║    $ tb db update --version 1.2.0    # Rolling cluster update              ║
║                                                                            ║
╚════════════════════════════════════════════════════════════════════════════╝
        """
    )

    subparsers = parser.add_subparsers(dest="action", required=False, help="Available actions")

    # Define common arguments
    instance_arg = {
        'name_or_flags': ['--instance-id'],
        'type': str,
        'help': 'Target a specific instance ID. If omitted, action applies to the whole cluster.',
        'default': None
    }
    version_arg = {
        'name_or_flags': ['--version'],
        'type': str,
        'help': 'Specify a version string for the executable (e.g., "1.2.0").',
        'default': 'dev'
    }

    # --- Define Commands ---
    p_start = subparsers.add_parser('start', help='Start database instance(s)')
    p_start.add_argument(*instance_arg['name_or_flags'],
                         **{k: v for k, v in instance_arg.items() if k != 'name_or_flags'})
    p_start.add_argument(*version_arg['name_or_flags'],
                         **{k: v for k, v in version_arg.items() if k != 'name_or_flags'})

    p_stop = subparsers.add_parser('stop', help='Stop database instance(s)')
    p_stop.add_argument(*instance_arg['name_or_flags'],
                        **{k: v for k, v in instance_arg.items() if k != 'name_or_flags'})

    p_status = subparsers.add_parser('status', help='Show running status of instance(s)')
    p_status.add_argument(*instance_arg['name_or_flags'],
                          **{k: v for k, v in instance_arg.items() if k != 'name_or_flags'})

    p_health = subparsers.add_parser('health', help='Perform health check on instance(s)')
    p_health.add_argument(*instance_arg['name_or_flags'],
                          **{k: v for k, v in instance_arg.items() if k != 'name_or_flags'})

    p_update = subparsers.add_parser('update', help='Perform rolling update on cluster')
    p_update.add_argument(*instance_arg['name_or_flags'],
                          **{k: v for k, v in instance_arg.items() if k != 'name_or_flags'})
    version_arg_update = {**version_arg, 'required': True}
    p_update.add_argument(*version_arg_update['name_or_flags'],
                          **{k: v for k, v in version_arg_update.items() if k != 'name_or_flags'})

    subparsers.add_parser('build', help='Build the Rust executable from source')
    subparsers.add_parser('clean', help='Clean the Rust build artifacts')
    subparsers.add_parser('discover', help='Interactive data discovery and manipulation')

    # --- Execute Command ---
    args = parser.parse_args()

    if args.action == 'build':
        handle_build()
        return
    if args.action == 'clean':
        handle_clean()
        return

    manager = ClusterManager()

    if args.action == 'discover':
        await handle_discover(manager)
        return
    executable_path = None
    if args.action in ['start', 'update']:
        executable_path = get_executable_path(update=(args.action == 'update'))
        if not executable_path:
            print_status(f"Could not find the {EXECUTABLE_NAME} executable", "error")
            print_status("Please build it first with: tb db build", "info")
            return

    if args.action == 'start':
        manager.start_all(executable_path, args.version, args.instance_id)
    elif args.action == 'stop':
        manager.stop_all(args.instance_id)
    elif args.action == 'status':
        manager.status_all(args.instance_id)
    elif args.action == 'health':
        manager.health_check_all(args.instance_id)
    elif args.action == 'update':
        manager.update_all_rolling(executable_path, args.version, args.instance_id)
    else:
        import asyncio
        await handle_discover(manager)

get_executable_path(base_name=EXECUTABLE_NAME, update=False)

Finds the release executable in standard locations.

Source code in toolboxv2/utils/clis/db_cli_manager.py

def get_executable_path(base_name: str = EXECUTABLE_NAME, update=False) -> Path | None:
    """Finds the release executable in standard locations."""
    name_with_ext = f"{base_name}.exe" if platform.system() == "Windows" else base_name
    from toolboxv2 import tb_root_dir
    search_paths = [
        tb_root_dir / "bin" / name_with_ext,
        tb_root_dir / "r_blob_db" / "target" / "release" / name_with_ext,
    ]
    if update:
        search_paths = search_paths[::-1]
    for path in search_paths:
        if path.is_file():
            return path.resolve()
    return None

handle_build()

Build the Rust project

Source code in toolboxv2/utils/clis/db_cli_manager.py

def handle_build():
    """Build the Rust project"""
    print_box_header("Building r_blob_db", "🔨")
    print_box_content("Compiling Rust project in release mode", "info")
    print_box_footer()

    from toolboxv2 import tb_root_dir

    try:
        with Spinner("Compiling with Cargo", symbols='t'):
            a,b = detect_shell()
            result = subprocess.run(
                [a,b,"cargo", "build", "--release", "--package", "r_blob_db"],
                check=True,
                cwd=tb_root_dir / "r_blob_db",
                capture_output=True,
                text=True
            )

        print_status("Build successful!", "success")

        exe_path = get_executable_path()
        if exe_path:
            bin_dir = tb_root_dir / "bin"
            bin_dir.mkdir(exist_ok=True)
            try:
                dest_path = bin_dir / exe_path.name
                shutil.copy(exe_path, dest_path)
                print_status(f"Executable copied to: {dest_path}", "info")
            except Exception as e:
                print_status(f"Warning: Failed to copy to bin: {e}", "warning")
                # Fallback to ubin
                ubin_dir = tb_root_dir / "ubin"
                ubin_dir.mkdir(exist_ok=True)
                dest_path = ubin_dir / exe_path.name
                try:
                    shutil.copy(exe_path, dest_path)
                    print_status(f"Copied to fallback location: {dest_path}", "info")
                except Exception as e_ubin:
                    print_status(f"Error copying to ubin: {e_ubin}", "error")

    except subprocess.CalledProcessError as e:
        print_status("Build failed!", "error")
        print(Style.GREY(e.stderr))
    except FileNotFoundError:
        print_status("Build failed: 'cargo' command not found", "error")
        print_status("Is Rust installed and in your PATH?", "info")

handle_clean()

Clean build artifacts

Source code in toolboxv2/utils/clis/db_cli_manager.py

def handle_clean():
    """Clean build artifacts"""
    print_box_header("Cleaning Build Artifacts", "🧹")
    print_box_footer()

    try:
        with Spinner("Running cargo clean", symbols='+'):
            a,b=detect_shell()
            from toolboxv2 import tb_root_dir
            subprocess.run([a,b,"cargo", "clean"], check=True, capture_output=True, cwd=tb_root_dir/"r_blob_db")
        print_status("Clean successful!", "success")
    except Exception as e:
        print_status(f"Clean failed: {e}", "error")

handle_discover(manager) async

Handle interactive data discovery

Source code in toolboxv2/utils/clis/db_cli_manager.py

async def handle_discover(manager: ClusterManager):
    """Handle interactive data discovery"""
    discovery = DataDiscovery(manager)
    await discovery.run()

tb_lang_cli

cli_tbx_main()

Main entry point for TB Language CLI

Source code in toolboxv2/utils/clis/tb_lang_cli.py

def cli_tbx_main():
    """Main entry point for TB Language CLI"""
    Copyparser = argparse.ArgumentParser(
        description="🚀 TB Language - Unified Multi-Language Programming Environment",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        prog='tb run',
        epilog="""
╔════════════════════════════════════════════════════════════════════════════╗
║                           Command Examples                                 ║
╠════════════════════════════════════════════════════════════════════════════╣
║                                                                            ║
║  Setup & Build:                                                            ║
║    $ tb run build                    # Build TB Language (native/release)  ║
║    $ tb run build --debug            # Build in debug mode                 ║
║    $ tb run build --target android   # Build for Android (all archs)       ║
║    $ tb run build --target ios       # Build for iOS (all archs)           ║
║    $ tb run build --target windows   # Cross-compile for Windows           ║
║    $ tb run build --target all       # Build for all platforms             ║
║    $ tb run clean                    # Clean build artifacts               ║
║                                                                            ║
║  Running Programs:                                                         ║
║    $ tb run x program.tb           # Run in JIT mode (default)             ║
║    $ tb run x program.tb --mode compiled                                   ║
║    $ tb run x program.tb --mode streaming                                  ║
║                                                                            ║
║  Compilation:                                                              ║
║    $ tb run compile input.tb output  # Compile to native                   ║
║    $ tb run compile app.tb app.wasm --target wasm                          ║
║                                                                            ║
║  Development:                                                              ║
║    $ tb run repl                     # Start interactive REPL              ║
║    $ tb run check program.tb         # Check syntax & types                ║
║    $ tb run examples                 # Browse and run examples             ║
║                                                                            ║
║  Project Management:                                                       ║
║    $ tb run init myproject           # Create new TB project               ║
║    $ tb run info                     # Show system information             ║
║                                                                            ║
║  Nested Tools:                                                             ║
║    $ tb run support [args]           # System support operations           ║
║    $ tb run ide [args]               # Language IDE extension tools        ║
║    $ tb run test [args]              # TB language testing and examples    ║
║                                                                            ║
╚════════════════════════════════════════════════════════════════════════════╝
"""
    )
    Copysubparsers = Copyparser.add_subparsers(dest="command", required=False)

    # Build command
    p_build = Copysubparsers.add_parser('build', help='Build TB Language executable')
    p_build.add_argument('--debug', action='store_true', help='Build in debug mode')
    p_build.add_argument('--target',
                        choices=['native', 'windows', 'linux', 'macos', 'macos-arm',
                                'android', 'ios', 'all'],
                        default='native',
                        help='Build target platform (default: native)')
    p_build.add_argument('--no-export', action='store_true',
                        help='Skip exporting to bin directory')

    # Clean command
    Copysubparsers.add_parser('clean', help='Clean build artifacts')

    # Run command
    p_run = Copysubparsers.add_parser('x', help='Run a TB program')
    p_run.add_argument('file', help='TB program file to run')
    p_run.add_argument('--mode', choices=['compiled', 'jit', 'streaming'],
                       default='jit', help='Execution mode')
    p_run.add_argument('--watch', action='store_true',
                       help='Watch for file changes and re-run')

    # Compile command
    p_compile = Copysubparsers.add_parser('compile', help='Compile TB program')
    p_compile.add_argument('input', help='Input TB file')
    p_compile.add_argument('output', help='Output file')
    p_compile.add_argument('--target', choices=['native', 'wasm', 'library'],
                           default='native', help='Compilation target')

    # REPL command
    Copysubparsers.add_parser('repl', help='Start interactive REPL')

    # Check command
    p_check = Copysubparsers.add_parser('check', help='Check syntax and types')
    p_check.add_argument('file', help='TB file to check')

    # Init command
    p_init = Copysubparsers.add_parser('init', help='Initialize new TB project')
    p_init.add_argument('name', help='Project name')

    # Examples command
    Copysubparsers.add_parser('examples', help='Browse and run examples')

    # Info command
    Copysubparsers.add_parser('info', help='Show system information')

    # System support command
    p_support = Copysubparsers.add_parser('support', help='System support operations')
    p_support.add_argument('support_args', nargs='*', help='Arguments for system support')

    # IDE extension command
    p_ide = Copysubparsers.add_parser('ide', help='Language IDE extension operations')
    p_ide.add_argument('ide_args', nargs='*', help='Arguments for IDE extension')

    # Test examples command
    p_test = Copysubparsers.add_parser('test', help='TB language testing and examples')
    p_test.add_argument('test_args', nargs='*', help='Arguments for testing')
    p_test.add_argument('--verbose', '-v', action='store_true', help='Verbose output')
    p_test.add_argument('--filter', help='Filter tests by name')
    p_test.add_argument('--failed', '-f', action='store_true', help='Run only failed tests')
    args = Copyparser.parse_args()

    # Execute command
    if args.command == 'build':
        success = handle_build(
            release=not args.debug,
            target=args.target,
            export_bin=not args.no_export
        )
    elif args.command == 'clean':
        success = handle_clean()
    elif args.command == 'x':
        success = handle_run(args.file, mode=args.mode, watch=args.watch)
    elif args.command == 'compile':
        success = handle_compile(args.input, args.output, target=args.target)
    elif args.command == 'repl':
        success = handle_repl()
    elif args.command == 'check':
        success = handle_check(args.file)
    elif args.command == 'init':
        success = handle_init(args.name)
    elif args.command == 'examples':
        success = handle_examples()
    elif args.command == 'info':
        handle_info()
        success = True
    elif args.command == 'support':
        success = handle_system_support(args.support_args)
    elif args.command == 'ide':
        success = handle_ide_extension(args.ide_args)
    elif args.command == 'test':
        success = handle_test_examples(args.test_args)
    else:
        # No command provided, show help
        Copyparser.print_help()
        success = True

    sys.exit(0 if success else 1)

detect_shell()

Detect shell for running commands

Source code in toolboxv2/utils/clis/tb_lang_cli.py

def detect_shell():
    """Detect shell for running commands"""
    if platform.system() == "Windows":
        return "powershell", "-Command"
    else:
        return "sh", "-c"

get_executable_path()

Find the compiled TB executable

Source code in toolboxv2/utils/clis/tb_lang_cli.py

def get_executable_path() -> Optional[Path]:
    """Find the compiled TB executable"""
    tb_root = get_tb_root()
    name_with_ext = f"{EXECUTABLE_NAME}.exe" if platform.system() == "Windows" else EXECUTABLE_NAME

    search_paths = [
        tb_root / "bin" / name_with_ext,
        get_project_dir() / "target" / "release" / name_with_ext,
        get_project_dir() / "target" / "debug" / name_with_ext,
    ]

    for path in search_paths:
        if path.is_file():
            return path.resolve()

    return None

get_project_dir()

Get the TB language project directory

Source code in toolboxv2/utils/clis/tb_lang_cli.py

def get_project_dir() -> Path:
    """Get the TB language project directory"""
    return get_tb_root() / PROJECT_DIR

get_tb_root()

Get the toolbox root directory

Source code in toolboxv2/utils/clis/tb_lang_cli.py

def get_tb_root() -> Path:
    """Get the toolbox root directory"""
    try:
        from toolboxv2 import tb_root_dir
        return tb_root_dir
    except ImportError:
        return Path(__file__).parent.parent.parent

handle_build(release=True, target='native', export_bin=True)

Build the TB language executable for various targets

Parameters:

Name	Type	Description	Default
release	bool	Build in release mode (default: True)	True
target	str	Build target - native, windows, linux, macos, android, ios, all (default: native)	'native'
export_bin	bool	Export binaries to bin directory (default: True)	True

Source code in toolboxv2/utils/clis/tb_lang_cli.py

def handle_build(release: bool = True, target: str = "native", export_bin: bool = True):
    """
    Build the TB language executable for various targets

    Args:
        release: Build in release mode (default: True)
        target: Build target - native, windows, linux, macos, android, ios, all (default: native)
        export_bin: Export binaries to bin directory (default: True)
    """
    print_box_header("Building TB Language", "🔨")
    print_box_content(f"Mode: {'Release' if release else 'Debug'}", "info")
    print_box_content(f"Target: {target}", "info")
    print_box_footer()

    project_dir = get_project_dir()

    if not project_dir.exists():
        print_status(f"Project directory not found: {project_dir}", "error")
        return False

    # Define target mappings
    desktop_targets = {
        "windows": "x86_64-pc-windows-msvc",
        "linux": "x86_64-unknown-linux-gnu",
        "macos": "x86_64-apple-darwin",
        "macos-arm": "aarch64-apple-darwin",
    }

    mobile_targets = {
        "android": ["aarch64-linux-android", "armv7-linux-androideabi",
                    "i686-linux-android", "x86_64-linux-android"],
        "ios": ["aarch64-apple-ios", "x86_64-apple-ios", "aarch64-apple-ios-sim"],
    }

    try:
        # Handle different target types
        if target == "native":
            # Build for current platform
            return _build_native(project_dir, release, export_bin)

        elif target in ["windows", "linux", "macos", "macos-arm"]:
            # Build for specific desktop platform
            return _build_desktop_target(project_dir, release, desktop_targets[target], export_bin)

        elif target == "android":
            # Build for all Android targets using mobile script
            return _build_mobile_platform(project_dir, release, "android", export_bin)

        elif target == "ios":
            # Build for all iOS targets using mobile script
            return _build_mobile_platform(project_dir, release, "ios", export_bin)

        elif target == "all":
            # Build for all platforms
            return _build_all_platforms(project_dir, release, export_bin)

        else:
            print_status(f"Unknown target: {target}", "error")
            return False

    except FileNotFoundError:
        print_status("Build failed: 'cargo' command not found", "error")
        print_status("Is Rust installed and in your PATH?", "info")
        print_status("Install from: https://rustup.rs", "info")
        return False
    except Exception as e:
        print_status(f"Build failed: {e}", "error")
        return False

handle_check(file_path)

Check a TB program without executing

Source code in toolboxv2/utils/clis/tb_lang_cli.py

def handle_check(file_path: str):
    """Check a TB program without executing"""
    exe_path = get_executable_path()

    if not exe_path:
        print_status("TB executable not found!", "error")
        return False

    if not Path(file_path).exists():
        print_status(f"File not found: {file_path}", "error")
        return False

    try:
        result = subprocess.run([str(exe_path), "check", file_path], check=True)
        return True
    except subprocess.CalledProcessError:
        return False
    except Exception as e:
        print_status(f"Failed to check: {e}", "error")
        return False

handle_clean()

Clean build artifacts

Source code in toolboxv2/utils/clis/tb_lang_cli.py

def handle_clean():
    """Clean build artifacts"""
    print_box_header("Cleaning Build Artifacts", "🧹")
    print_box_footer()

    project_dir = get_project_dir()

    try:
        shell, shell_flag = detect_shell()

        with Spinner("Running cargo clean", symbols='+'):
            subprocess.run(
                [shell, shell_flag, "cargo clean"],
                cwd=project_dir,
                capture_output=True,
                check=True
            )

        print_status("Clean successful!", "success")
        return True
    except Exception as e:
        print_status(f"Clean failed: {e}", "error")
        return False

handle_compile(input_file, output_file, target='native')

Compile a TB program

Source code in toolboxv2/utils/clis/tb_lang_cli.py

def handle_compile(input_file: str, output_file: str, target: str = "native"):
    """Compile a TB program"""
    exe_path = get_executable_path()

    if not exe_path:
        print_status("TB executable not found!", "error")
        return False

    if not Path(input_file).exists():
        print_status(f"Input file not found: {input_file}", "error")
        return False

    print_box_header("Compiling TB Program", "⚙️")
    print_box_content(f"Input: {input_file}", "info")
    print_box_content(f"Output: {output_file}", "info")
    print_box_content(f"Target: {target}", "info")
    print_box_footer()

    try:
        cmd = [str(exe_path), "compile", input_file, output_file, "--target", target]

        result = subprocess.run(cmd, check=True)

        print()
        print_status("Compilation successful!", "success")
        return True

    except subprocess.CalledProcessError:
        print()
        print_status("Compilation failed", "error")
        return False
    except Exception as e:
        print_status(f"Failed to compile: {e}", "error")
        return False

handle_examples()

Run example programs

Source code in toolboxv2/utils/clis/tb_lang_cli.py

def handle_examples():
    """Run example programs"""
    examples_dir = get_project_dir() / "examples"
    if not examples_dir.exists():
        print_status("Examples directory not found", "error")
        return False

    examples = list(examples_dir.glob("*.tb"))

    if not examples:
        print_status("No example files found", "warning")
        return False

    print_box_header("TB Language Examples", "📚")
    print()

    for i, example in enumerate(examples, 1):
        print(f"  {i}. {example.name}")

    print()
    print_box_footer()

    try:
        choice = input("Select example (number) or 'q' to quit: ").strip()

        if choice.lower() == 'q':
            return True

        idx = int(choice) - 1
        if 0 <= idx < len(examples):
            print()
            return handle_run(str(examples[idx]), mode="jit")
        else:
            print_status("Invalid selection", "error")
            return False

    except ValueError:
        print_status("Invalid input", "error")
        return False
    except KeyboardInterrupt:
        print()
        return True

handle_ide_extension(args)

Handle language IDE extension operations

Source code in toolboxv2/utils/clis/tb_lang_cli.py

def handle_ide_extension(args):
    """Handle language IDE extension operations"""
    return language_ide_extension(args)

handle_info()

Show system information

Source code in toolboxv2/utils/clis/tb_lang_cli.py

def handle_info():
    """Show system information"""
    print_box_header("TB Language System Information", "ℹ️")
    print()
    # TB Root
    tb_root = get_tb_root()
    print(f"  TB Root:     {tb_root}")

    # Project directory
    project_dir = get_project_dir()
    print(f"  Project Dir: {project_dir}")
    print(f"  Exists:      {project_dir.exists()}")

    # Executable
    exe_path = get_executable_path()
    if exe_path:
        print(f"  Executable:  {exe_path}")
        print(f"  Exists:      {exe_path.exists()}")
    else:
        print(f"  Executable:  Not found (build first)")

    # Rust toolchain
    print()
    print("  Rust Toolchain:")
    try:
        result = subprocess.run(
            ["rustc", "--version"],
            capture_output=True,
            text=True,
            check=True
        )
        print(f"    {result.stdout.strip()}")

        result = subprocess.run(
            ["cargo", "--version"],
            capture_output=True,
            text=True,
            check=True
        )
        print(f"    {result.stdout.strip()}")
    except FileNotFoundError:
        print(Style.RED("    Rust not found! Install from https://rustup.rs"))
    except subprocess.CalledProcessError:
        print(Style.RED("    Failed to get Rust version"))

    print()
    print_box_footer()

handle_init(project_name)

Initialize a new TB project

Source code in toolboxv2/utils/clis/tb_lang_cli.py

def handle_init(project_name: str):
    """Initialize a new TB project"""
    print_box_header(f"Creating TB Project: {project_name}", "📦")
    print_box_footer()

    from toolboxv2 import tb_root_dir, init_cwd

    if init_cwd == tb_root_dir:
        print_status("Cannot create project in TB root directory", "error")
        return False

    project_path = init_cwd / project_name

    if project_path.exists():
        print_status(f"Directory already exists: {project_path}", "error")
        return False

    try:
        # Create directory structure
        project_path.mkdir()
        (project_path / "src").mkdir()
        (project_path / "examples").mkdir()

        # Create main.tb
        main_tb = project_path / "src" / "main.tb"
        main_tb.write_text('''#!tb
@config {
    mode: "jit"
    type_mode: "static"
    optimize: true
}

@shared {
    app_name: "''' + project_name + '''"
}

fn main() {
    echo "Hello from $app_name!"
}

main()
''')

        # Create README
        readme = project_path / "README.md"
        readme.write_text(f'''# {project_name}

A TB Language project.

## Running


```bash
tb run x src/main.tb
Building
bash
tb compile src/main.tb bin/{project_name}
''')
        print_status(f"✓ Created project structure", "success")
        print_status(f"✓ Created src/main.tb", "success")
        print_status(f"✓ Created README.md", "success")
        print()
        print_status(f"Get started with:", "info")
        print(f"  cd {project_name}")
        print(f"  tb run src/main.tb")

        return True

    except Exception as e:
        print_status(f"Failed to create project: {e}", "error")
        return False

handle_repl()

Start TB REPL

Source code in toolboxv2/utils/clis/tb_lang_cli.py

def handle_repl():
    """Start TB REPL"""
    exe_path = get_executable_path()

    if not exe_path:
        print_status("TB executable not found!", "error")
        return False

    try:
        subprocess.run([str(exe_path), "repl"])
        return True
    except KeyboardInterrupt:
        print()
        return True
    except Exception as e:
        print_status(f"Failed to start REPL: {e}", "error")
        return False

handle_run(file_path, mode='jit', watch=False)

Run a TB program

Source code in toolboxv2/utils/clis/tb_lang_cli.py

def handle_run(file_path: str, mode: str = "jit", watch: bool = False):
    """Run a TB program"""
    exe_path = get_executable_path()

    if not exe_path:
        print_status("TB executable not found!", "error")
        print_status("Build it first with: tb x build", "info")
        return False

    if not Path(file_path).exists():
        print_status(f"File not found: {file_path}", "error")
        return False

    print_box_header(f"Running TB Program", "🚀")
    print_box_content(f"File: {file_path}", "info")
    print_box_content(f"Mode: {mode}", "info")
    print_box_footer()

    try:
        cmd = [str(exe_path), "run", file_path, "--mode", mode]

        result = subprocess.run(cmd, check=False)

        if result.returncode == 0:
            print()
            print_status("Execution completed successfully", "success")
            return True
        else:
            print()
            print_status(f"Execution failed with code {result.returncode}", "error")
            return False

    except KeyboardInterrupt:
        print()
        print_status("Execution interrupted", "warning")
        return False
    except Exception as e:
        print_status(f"Failed to run: {e}", "error")
        return False

handle_system_support(args)

Handle system support operations

Source code in toolboxv2/utils/clis/tb_lang_cli.py

def handle_system_support(args):
    """Handle system support operations"""
    return system_tbx_support(*args)

handle_test_examples(args)

Handle TB language testing and examples

Source code in toolboxv2/utils/clis/tb_lang_cli.py

def handle_test_examples(args):
    """Handle TB language testing and examples"""
    return test_tbx_examples(args)

tcm_p2p_cli

ChatListener

Background thread to listen for new chat messages.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py

class ChatListener(threading.Thread):
    """Background thread to listen for new chat messages."""

    def __init__(self, chat_manager, room_id, password, callback):
        super().__init__(daemon=True)
        self.chat_manager = chat_manager
        self.room_id = room_id
        self.password = password
        self.callback = callback
        self.running = True
        self.last_message_count = 0

    def run(self):
        while self.running:
            try:
                result = self.chat_manager.get_messages(self.room_id, self.password, 50)
                if result.is_ok():
                    messages = result.get()
                    if len(messages) > self.last_message_count:
                        # New messages arrived
                        new_messages = messages[self.last_message_count:]
                        for msg in new_messages:
                            if not msg['is_own']:  # Only show messages from others
                                self.callback(msg)
                        self.last_message_count = len(messages)
            except Exception:
                pass
            time.sleep(1)  # Poll every second

    def stop(self):
        self.running = False

ChatMessage dataclass

Represents a chat message with encryption support.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py

@dataclass
class ChatMessage:
    """Represents a chat message with encryption support."""
    sender: str
    content: str
    timestamp: datetime
    room_id: str
    message_type: MessageType = MessageType.TEXT
    encrypted: bool = True
    file_name: Optional[str] = None
    file_size: Optional[int] = None

    def to_dict(self) -> dict:
        return {
            **asdict(self),
            'timestamp': self.timestamp.isoformat(),
            'message_type': self.message_type.value
        }

    @classmethod
    def from_dict(cls, data: dict) -> 'ChatMessage':
        data['timestamp'] = datetime.fromisoformat(data['timestamp'])
        data['message_type'] = MessageType(data['message_type'])
        return cls(**data)

ChatRoom dataclass

Represents a P2P chat room with E2E encryption.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py

@dataclass
class ChatRoom:
    """Represents a P2P chat room with E2E encryption."""
    room_id: str
    name: str
    owner: str
    participants: Set[str]
    is_locked: bool
    is_private: bool
    created_at: datetime
    encryption_key: str
    max_participants: int = 10
    voice_enabled: bool = False
    file_transfer_enabled: bool = True

    def to_dict(self) -> dict:
        return {
            **asdict(self),
            'participants': list(self.participants),
            'created_at': self.created_at.isoformat()
        }

    @classmethod
    def from_dict(cls, data: dict) -> 'ChatRoom':
        data['participants'] = set(data['participants'])
        data['created_at'] = datetime.fromisoformat(data['created_at'])
        return cls(**data)

CryptoManager

Handles all E2E encryption operations.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py

class CryptoManager:
    """Handles all E2E encryption operations."""

    @staticmethod
    def generate_room_key(room_id: str, password: str) -> bytes:
        """Generate encryption key for room."""
        salt = room_id.encode()
        kdf = PBKDF2HMAC(
            algorithm=hashes.SHA256(),
            length=32,
            salt=salt,
            iterations=100000,
        )
        return base64.urlsafe_b64encode(kdf.derive(password.encode()))

    @staticmethod
    def encrypt_message(message: str, key: bytes) -> str:
        """Encrypt message content."""
        f = Fernet(key)
        return f.encrypt(message.encode()).decode()

    @staticmethod
    def decrypt_message(encrypted_message: str, key: bytes) -> str:
        """Decrypt message content."""
        f = Fernet(key)
        return f.decrypt(encrypted_message.encode()).decode()

    @staticmethod
    def encrypt_file(file_path: Path, key: bytes) -> bytes:
        """Encrypt file content."""
        f = Fernet(key)
        with open(file_path, 'rb') as file:
            return f.encrypt(file.read())

    @staticmethod
    def decrypt_file(encrypted_data: bytes, key: bytes, output_path: Path):
        """Decrypt file content."""
        f = Fernet(key)
        decrypted_data = f.decrypt(encrypted_data)
        with open(output_path, 'wb') as file:
            file.write(decrypted_data)

    @staticmethod
    def encrypt_bytes(data: bytes, key: bytes) -> bytes:
        """Encrypt binary data directly (for audio/files)."""
        f = Fernet(key)
        return f.encrypt(data)

    @staticmethod
    def decrypt_bytes(encrypted_data: bytes, key: bytes) -> bytes:
        """Decrypt binary data directly (for audio/files)."""
        f = Fernet(key)
        return f.decrypt(encrypted_data)

decrypt_bytes(encrypted_data, key) staticmethod

Decrypt binary data directly (for audio/files).

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py

@staticmethod
def decrypt_bytes(encrypted_data: bytes, key: bytes) -> bytes:
    """Decrypt binary data directly (for audio/files)."""
    f = Fernet(key)
    return f.decrypt(encrypted_data)

decrypt_file(encrypted_data, key, output_path) staticmethod

Decrypt file content.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py

@staticmethod
def decrypt_file(encrypted_data: bytes, key: bytes, output_path: Path):
    """Decrypt file content."""
    f = Fernet(key)
    decrypted_data = f.decrypt(encrypted_data)
    with open(output_path, 'wb') as file:
        file.write(decrypted_data)

decrypt_message(encrypted_message, key) staticmethod

Decrypt message content.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py

@staticmethod
def decrypt_message(encrypted_message: str, key: bytes) -> str:
    """Decrypt message content."""
    f = Fernet(key)
    return f.decrypt(encrypted_message.encode()).decode()

encrypt_bytes(data, key) staticmethod

Encrypt binary data directly (for audio/files).

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py

@staticmethod
def encrypt_bytes(data: bytes, key: bytes) -> bytes:
    """Encrypt binary data directly (for audio/files)."""
    f = Fernet(key)
    return f.encrypt(data)

encrypt_file(file_path, key) staticmethod

Encrypt file content.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py

@staticmethod
def encrypt_file(file_path: Path, key: bytes) -> bytes:
    """Encrypt file content."""
    f = Fernet(key)
    with open(file_path, 'rb') as file:
        return f.encrypt(file.read())

encrypt_message(message, key) staticmethod

Encrypt message content.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py

@staticmethod
def encrypt_message(message: str, key: bytes) -> str:
    """Encrypt message content."""
    f = Fernet(key)
    return f.encrypt(message.encode()).decode()

generate_room_key(room_id, password) staticmethod

Generate encryption key for room.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py

@staticmethod
def generate_room_key(room_id: str, password: str) -> bytes:
    """Generate encryption key for room."""
    salt = room_id.encode()
    kdf = PBKDF2HMAC(
        algorithm=hashes.SHA256(),
        length=32,
        salt=salt,
        iterations=100000,
    )
    return base64.urlsafe_b64encode(kdf.derive(password.encode()))

EnhancedInstanceManager

Enhanced instance manager with chat integration.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py

class EnhancedInstanceManager:
    """Enhanced instance manager with chat integration."""

    def __init__(self, name: str, app: App):
        self.name = name
        self.app = app
        self.instance_dir = INSTANCES_ROOT_DIR / self.name
        self.state_file = self.instance_dir / "state.json"
        self.config_file = self.instance_dir / "config.toml"
        self.log_file = self.instance_dir / "instance.log"

    def read_state(self) -> dict:
        """Read instance state."""
        if not self.state_file.exists():
            return {}
        try:
            with open(self.state_file) as f:
                return json.load(f)
        except (json.JSONDecodeError, FileNotFoundError):
            return {}

    def write_state(self, state_data: dict):
        """Write instance state."""
        self.instance_dir.mkdir(parents=True, exist_ok=True)
        with open(self.state_file, 'w') as f:
            json.dump(state_data, f, indent=2)

    def is_running(self) -> bool:
        """Check if instance is running."""
        pid = self.read_state().get('pid')
        return psutil.pid_exists(pid) if pid else False

    def generate_config(self, mode: str, config_data: dict):
        """Generate config.toml for instance."""
        content = f'mode = "{mode}"\n\n'

        if mode == "relay":
            content += "[relay]\n"
            content += f'bind_address = "{config_data.get("bind_address", "0.0.0.0:9000")}"\n'
            content += f'password = "{config_data.get("password", "")}"\n'

        elif mode == "peer":
            content += "[peer]\n"
            content += f'relay_address = "{config_data.get("relay_address", "127.0.0.1:9000")}"\n'
            content += f'relay_password = "{config_data.get("relay_password", "")}"\n'
            content += f'peer_id = "{config_data.get("peer_id", "default-peer")}"\n'
            content += f'listen_address = "{config_data.get("listen_address", "127.0.0.1:8000")}"\n'
            content += f'forward_to_address = "{config_data.get("forward_to_address", "127.0.0.1:3000")}"\n'
            if config_data.get("target_peer_id"):
                content += f'target_peer_id = "{config_data.get("target_peer_id")}"\n'

        self.instance_dir.mkdir(parents=True, exist_ok=True)
        with open(self.config_file, "w") as f:
            f.write(content)

    def start(self, executable_path: Path, mode: str, config_data: dict, chat_room: Optional[str] = None) -> bool:
        """Start instance."""
        if self.is_running():
            print(Style.YELLOW(f"Instance '{self.name}' is already running"))
            return True

        self.generate_config(mode, config_data)
        log_handle = open(self.log_file, 'a')

        try:
            with Spinner(f"Starting '{self.name}'", symbols="d"):
                process = subprocess.Popen(
                    [str(executable_path)],
                    cwd=str(self.instance_dir),
                    stdout=log_handle,
                    stderr=log_handle,
                    creationflags=subprocess.DETACHED_PROCESS if platform.system() == "Windows" else 0
                )
                time.sleep(1.5)

            if process.poll() is not None:
                print(f"\n{Style.RED2('❌')} Instance failed to start")
                return False

            state = {'pid': process.pid, 'mode': mode, 'config': config_data}
            if chat_room:
                state['chat_room'] = chat_room
            self.write_state(state)

            print(f"\n{Style.GREEN2('✅')} Instance '{Style.Bold(self.name)}' started (PID: {process.pid})")
            if chat_room:
                print(f"   {Style.BLUE('Chat Room:')} {Style.CYAN(chat_room)}")
            return True

        except Exception as e:
            print(f"\n{Style.RED2('❌')} Failed to start: {e}")
            return False

    def stop(self, timeout: int = 10) -> bool:
        """Stop instance."""
        if not self.is_running():
            self.write_state({})
            return True

        pid = self.read_state().get('pid')

        try:
            with Spinner(f"Stopping '{self.name}'", symbols="+", time_in_s=timeout, count_down=True):
                proc = psutil.Process(pid)
                proc.terminate()
                proc.wait(timeout)
        except psutil.TimeoutExpired:
            proc.kill()
        except (psutil.NoSuchProcess, Exception):
            pass

        self.write_state({})
        print(f"\n{Style.VIOLET2('⏹️')} Instance '{Style.Bold(self.name)}' stopped")
        return True

generate_config(mode, config_data)

Generate config.toml for instance.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py

def generate_config(self, mode: str, config_data: dict):
    """Generate config.toml for instance."""
    content = f'mode = "{mode}"\n\n'

    if mode == "relay":
        content += "[relay]\n"
        content += f'bind_address = "{config_data.get("bind_address", "0.0.0.0:9000")}"\n'
        content += f'password = "{config_data.get("password", "")}"\n'

    elif mode == "peer":
        content += "[peer]\n"
        content += f'relay_address = "{config_data.get("relay_address", "127.0.0.1:9000")}"\n'
        content += f'relay_password = "{config_data.get("relay_password", "")}"\n'
        content += f'peer_id = "{config_data.get("peer_id", "default-peer")}"\n'
        content += f'listen_address = "{config_data.get("listen_address", "127.0.0.1:8000")}"\n'
        content += f'forward_to_address = "{config_data.get("forward_to_address", "127.0.0.1:3000")}"\n'
        if config_data.get("target_peer_id"):
            content += f'target_peer_id = "{config_data.get("target_peer_id")}"\n'

    self.instance_dir.mkdir(parents=True, exist_ok=True)
    with open(self.config_file, "w") as f:
        f.write(content)

is_running()

Check if instance is running.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py

def is_running(self) -> bool:
    """Check if instance is running."""
    pid = self.read_state().get('pid')
    return psutil.pid_exists(pid) if pid else False

read_state()

Read instance state.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py

def read_state(self) -> dict:
    """Read instance state."""
    if not self.state_file.exists():
        return {}
    try:
        with open(self.state_file) as f:
            return json.load(f)
    except (json.JSONDecodeError, FileNotFoundError):
        return {}

start(executable_path, mode, config_data, chat_room=None)

Start instance.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py

def start(self, executable_path: Path, mode: str, config_data: dict, chat_room: Optional[str] = None) -> bool:
    """Start instance."""
    if self.is_running():
        print(Style.YELLOW(f"Instance '{self.name}' is already running"))
        return True

    self.generate_config(mode, config_data)
    log_handle = open(self.log_file, 'a')

    try:
        with Spinner(f"Starting '{self.name}'", symbols="d"):
            process = subprocess.Popen(
                [str(executable_path)],
                cwd=str(self.instance_dir),
                stdout=log_handle,
                stderr=log_handle,
                creationflags=subprocess.DETACHED_PROCESS if platform.system() == "Windows" else 0
            )
            time.sleep(1.5)

        if process.poll() is not None:
            print(f"\n{Style.RED2('❌')} Instance failed to start")
            return False

        state = {'pid': process.pid, 'mode': mode, 'config': config_data}
        if chat_room:
            state['chat_room'] = chat_room
        self.write_state(state)

        print(f"\n{Style.GREEN2('✅')} Instance '{Style.Bold(self.name)}' started (PID: {process.pid})")
        if chat_room:
            print(f"   {Style.BLUE('Chat Room:')} {Style.CYAN(chat_room)}")
        return True

    except Exception as e:
        print(f"\n{Style.RED2('❌')} Failed to start: {e}")
        return False

stop(timeout=10)

Stop instance.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py

def stop(self, timeout: int = 10) -> bool:
    """Stop instance."""
    if not self.is_running():
        self.write_state({})
        return True

    pid = self.read_state().get('pid')

    try:
        with Spinner(f"Stopping '{self.name}'", symbols="+", time_in_s=timeout, count_down=True):
            proc = psutil.Process(pid)
            proc.terminate()
            proc.wait(timeout)
    except psutil.TimeoutExpired:
        proc.kill()
    except (psutil.NoSuchProcess, Exception):
        pass

    self.write_state({})
    print(f"\n{Style.VIOLET2('⏹️')} Instance '{Style.Bold(self.name)}' stopped")
    return True

write_state(state_data)

Write instance state.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py

def write_state(self, state_data: dict):
    """Write instance state."""
    self.instance_dir.mkdir(parents=True, exist_ok=True)
    with open(self.state_file, 'w') as f:
        json.dump(state_data, f, indent=2)

FileTransferManager

Manages P2P file transfers with E2E encryption.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py

class FileTransferManager:
    """Manages P2P file transfers with E2E encryption."""

    def __init__(self, room_id: str, encryption_key: bytes):
        self.room_id = room_id
        self.encryption_key = encryption_key
        self.transfer_dir = FILE_TRANSFER_DIR / room_id
        self.transfer_dir.mkdir(parents=True, exist_ok=True)

    def prepare_file(self, file_path: Path) -> Tuple[str, int]:
        """Prepare file for transfer (encrypt and chunk)."""
        if not file_path.exists():
            raise FileNotFoundError(f"File not found: {file_path}")

        file_size = file_path.stat().st_size
        if file_size > MAX_FILE_SIZE:
            raise ValueError(f"File too large: {file_size} bytes (max: {MAX_FILE_SIZE})")

        # Encrypt file
        encrypted_data = CryptoManager.encrypt_file(file_path, self.encryption_key)

        # Save encrypted file
        transfer_id = hashlib.sha256(f"{file_path.name}{time.time()}".encode()).hexdigest()[:16]
        encrypted_file_path = self.transfer_dir / f"{transfer_id}.enc"

        with open(encrypted_file_path, 'wb') as f:
            f.write(encrypted_data)

        return transfer_id, len(encrypted_data)

    def receive_file(self, transfer_id: str, file_name: str) -> Path:
        """Receive and decrypt file."""
        encrypted_file_path = self.transfer_dir / f"{transfer_id}.enc"

        if not encrypted_file_path.exists():
            raise FileNotFoundError(f"Transfer file not found: {transfer_id}")

        # Read encrypted data
        with open(encrypted_file_path, 'rb') as f:
            encrypted_data = f.read()

        # Decrypt and save
        output_path = FILE_TRANSFER_DIR / "received" / file_name
        output_path.parent.mkdir(parents=True, exist_ok=True)

        CryptoManager.decrypt_file(encrypted_data, self.encryption_key, output_path)

        return output_path

prepare_file(file_path)

Prepare file for transfer (encrypt and chunk).

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py

def prepare_file(self, file_path: Path) -> Tuple[str, int]:
    """Prepare file for transfer (encrypt and chunk)."""
    if not file_path.exists():
        raise FileNotFoundError(f"File not found: {file_path}")

    file_size = file_path.stat().st_size
    if file_size > MAX_FILE_SIZE:
        raise ValueError(f"File too large: {file_size} bytes (max: {MAX_FILE_SIZE})")

    # Encrypt file
    encrypted_data = CryptoManager.encrypt_file(file_path, self.encryption_key)

    # Save encrypted file
    transfer_id = hashlib.sha256(f"{file_path.name}{time.time()}".encode()).hexdigest()[:16]
    encrypted_file_path = self.transfer_dir / f"{transfer_id}.enc"

    with open(encrypted_file_path, 'wb') as f:
        f.write(encrypted_data)

    return transfer_id, len(encrypted_data)

receive_file(transfer_id, file_name)

Receive and decrypt file.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py

def receive_file(self, transfer_id: str, file_name: str) -> Path:
    """Receive and decrypt file."""
    encrypted_file_path = self.transfer_dir / f"{transfer_id}.enc"

    if not encrypted_file_path.exists():
        raise FileNotFoundError(f"Transfer file not found: {transfer_id}")

    # Read encrypted data
    with open(encrypted_file_path, 'rb') as f:
        encrypted_data = f.read()

    # Decrypt and save
    output_path = FILE_TRANSFER_DIR / "received" / file_name
    output_path.parent.mkdir(parents=True, exist_ok=True)

    CryptoManager.decrypt_file(encrypted_data, self.encryption_key, output_path)

    return output_path

InteractiveP2PCLI

Interactive P2P CLI with modern ToolBox-style interface.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py

class InteractiveP2PCLI:
    """Interactive P2P CLI with modern ToolBox-style interface."""

    def __init__(self):
        self.app = get_app("P2P_Interactive_CLI")

        self.voice_server_info: Dict[str, Tuple[str, int]] = {}
        self.chat_manager = P2PChatManager(self.app, self.voice_server_info)
        self.instances: Dict[str, EnhancedInstanceManager] = {}
        self.current_chat_room = None
        self.current_chat_password = None
        self.running = True
        self._load_instances()

        self.file_managers: Dict[str, FileTransferManager] = {}
        self.voice_manager: Optional[VoiceChatManager] = None

    def _load_instances(self):
        """Load existing instances."""
        if INSTANCES_ROOT_DIR.exists():
            for instance_dir in INSTANCES_ROOT_DIR.iterdir():
                if instance_dir.is_dir():
                    self.instances[instance_dir.name] = EnhancedInstanceManager(instance_dir.name, self.app)

    def clear_screen(self):
        """Clear terminal screen."""
        os.system('cls' if os.name == 'nt' else 'clear')

    def print_header(self):
        """Print main header."""
        print(f"""
{Style.CYAN('╔══════════════════════════════════════════════════════════════════════╗')}
{Style.CYAN('║')} {Style.Bold(Style.WHITE('🌐 ToolBox P2P Manager'))} {Style.CYAN('v2.0')} {Style.GREY('- Interactive Mode')} {self._current_room_name() or '':<21} {Style.CYAN('║')}
{Style.CYAN('║')} {Style.GREY('E2E Encrypted Chat • File Transfer • Voice Chat • P2P Tunnels')} {' ' * 6} {Style.CYAN('║')}
{Style.CYAN('╚══════════════════════════════════════════════════════════════════════╝')}
""")

    def print_menu(self):
        """Print main menu."""
        print(f"""
{Style.Bold(Style.WHITE('┌─ 🎯 MAIN MENU ───────────────────────────────────────────────────────┐'))}
{Style.WHITE('│')}                                                                      {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('1.')} {Style.WHITE('💬 Chat Mode')}          - Start interactive E2E encrypted chat     {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('2.')} {Style.WHITE('🔧 P2P Configuration')}  - Configure P2P connections                {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('3.')} {Style.WHITE('📊 Status & Monitoring')} - View connections and rooms              {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('4.')} {Style.WHITE('⚙️  Settings')}           - Manage configuration                    {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('0.')} {Style.WHITE('🚪 Exit')}               - Quit application                         {Style.WHITE('│')}
{Style.WHITE('│')}                                                                      {Style.WHITE('│')}
{Style.Bold(Style.WHITE('└──────────────────────────────────────────────────────────────────────┘'))}
""")

    def chat_menu(self):
        """Interactive chat menu."""
        try:
            while True:
                self.clear_screen()
                self.print_header()

                # Show current room info
                if self.current_chat_room:
                    room = self.chat_manager.rooms.get(self.current_chat_room)
                    if room:
                        print(f"""
    {Style.GREEN('╔══ Current Room ════════════════════════════════════════════════════╗')}
    {Style.GREEN('║')} {Style.WHITE('Name:')} {Style.YELLOW(room.name):<30} {Style.WHITE('ID:')} {Style.CYAN(room.room_id):<15} {" "*22+Style.GREEN('║')}
    {Style.GREEN('║')} {Style.WHITE('Participants:')} {', '.join(list(room.participants)[:10]):<50}{'...' if len(room.participants) > 3 else '':<30}
    {Style.GREEN('╚════════════════════════════════════════════════════════════════════╝')}
    """)

                print(f"""
    {Style.Bold(Style.WHITE('┌─ 💬 CHAT MENU ───────────────────────────────────────────────────────┐'))}
    {Style.WHITE('│')}                                                                      {Style.WHITE('│')}
    {Style.WHITE('│')}  {Style.CYAN('1.')} {Style.WHITE('Create Room')}         - Create new E2E encrypted chat room         {Style.WHITE('│')}
    {Style.WHITE('│')}  {Style.CYAN('2.')} {Style.WHITE('Join Room')}           - Join existing room by ID                   {Style.WHITE('│')}
    {Style.WHITE('│')}  {Style.CYAN('3.')} {Style.WHITE('List Rooms')}          - Show available chat rooms                  {Style.WHITE('│')}
    {Style.WHITE('│')}  {Style.CYAN('4.')} {Style.WHITE('Interactive Chat')}    - Start live chat (current room)             {Style.WHITE('│')}
    {Style.WHITE('│')}  {Style.CYAN('5.')} {Style.WHITE('Send File')}           - Transfer file (E2E encrypted)              {Style.WHITE('│')}
    {Style.WHITE('│')}  {Style.CYAN('6.')} {Style.WHITE('Voice Chat')}          - Start voice chat (beta)                    {Style.WHITE('│')}
    {Style.WHITE('│')}  {Style.CYAN('7.')} {Style.WHITE('Lock Room')}           - Lock current room (owner only)             {Style.WHITE('│')}
    {Style.WHITE('│')}  {Style.CYAN('8.')} {Style.WHITE('Leave Room')}          - Leave current chat room                    {Style.WHITE('│')}
    {Style.WHITE('│')}  {Style.CYAN('0.')} {Style.WHITE('Back')}                - Return to main menu                        {Style.WHITE('│')}
    {Style.WHITE('│')}                                                                      {Style.WHITE('│')}
    {Style.Bold(Style.WHITE('└──────────────────────────────────────────────────────────────────────┘'))}
    """)

                choice = input(f"\n{Style.CYAN('❯')} {Style.WHITE('Select option:')} ").strip()

                if choice == '0':
                    break
                elif choice == '1':
                    self._create_chat_room()
                elif choice == '2':
                    self._join_chat_room()
                elif choice == '3':
                    self._list_chat_rooms()
                elif choice == '4':
                    self._interactive_chat()
                elif choice == '5':
                    self._send_file()
                elif choice == '6':
                    self._voice_chat()
                elif choice == '7':
                    self._lock_room()
                elif choice == '8':
                    self._leave_room()
                else:
                    print(f"{Style.RED('Invalid option')}")
                    time.sleep(1)
        finally:
            if self._current_room_name() is not None:
                self._leave_room(auto=True)

    def _create_chat_room(self):
        """Create new chat room."""
        print(f"\n{Style.Bold(Style.CYAN('Create New Chat Room'))}")
        print(Style.GREY('─' * 70))

        name = input(f"{Style.WHITE('Room name:')} ").strip()
        if not name:
            return

        password = input(f"{Style.WHITE('Room password:')} ").strip()
        if not password:
            return

        max_participants = input(f"{Style.WHITE('Max participants (default 10):')} ").strip()
        max_participants = int(max_participants) if max_participants.isdigit() else 10

        voice_enabled = input(f"{Style.WHITE('Enable voice chat? (y/N):')} ").strip().lower() == 'y'
        private = input(f"{Style.WHITE('Make private? (y/N):')} ").strip().lower() == 'y'

        result = self.chat_manager.create_room(name, password, max_participants, voice_enabled, private)

        if result.is_ok():
            data = result.get()
            print(f"\n{Style.GREEN2('✅ Room created successfully!')}")
            print(f"   {Style.WHITE('Room ID:')} {Style.CYAN(data['room_id'])}")
            print(f"   {Style.WHITE('Name:')} {Style.YELLOW(data['name'])}")

            # Auto-join created room
            self.current_chat_room = data['room_id']
            self.current_chat_password = password
        else:
            print(f"{Style.RED2('❌ Failed:')} {result.info}")

        input(f"\n{Style.GREY('Press Enter to continue...')}")

    def _join_chat_room(self):
        """Join existing chat room."""
        print(f"\n{Style.Bold(Style.CYAN('Join Chat Room'))}")
        print(Style.GREY('─' * 70))

        # First show available rooms
        result = self.chat_manager.list_rooms(show_all=True)
        if result.is_ok():
            rooms = result.get()
            if rooms:
                print(f"\n{Style.WHITE('Available Rooms:')}")
                for i, room in enumerate(rooms, 1):
                    status = "🔒" if room['is_locked'] else "🔓"
                    member = "✓" if room['is_member'] else " "
                    print(
                        f"  {i}. [{member}] {status} {Style.YELLOW(room['name'][:20])} - {Style.CYAN(room['room_id'])}")
                print()

        room_id = input(f"{Style.WHITE('Room ID:')} ").strip()
        if not room_id:
            return

        password = input(f"{Style.WHITE('Password:')} ").strip()
        if not password:
            return

        result = self.chat_manager.join_room(room_id, password)

        if result.is_ok():
            data = result.get()
            self.current_chat_room = room_id
            self.current_chat_password = password

            print(f"\n{Style.GREEN2('✅ Joined room successfully!')}")
            print(f"   {Style.WHITE('Room:')} {Style.YELLOW(data['name'])}")
            print(f"   {Style.WHITE('Participants:')} {', '.join(data['participants'])}")
            if data['voice_enabled']:
                print(f"   {Style.WHITE('Voice chat:')} {Style.GREEN('Enabled')}")
        else:
            print(f"{Style.RED2('❌ Failed:')} {result.info}")

        input(f"\n{Style.GREY('Press Enter to continue...')}")

    def _list_chat_rooms(self):
        """List available chat rooms."""
        print(f"\n{Style.Bold(Style.CYAN('Chat Rooms'))}")
        print(Style.GREY('═' * 90))

        result = self.chat_manager.list_rooms(show_all=True)

        if result.is_ok():
            rooms = result.get()
            if not rooms:
                print(Style.YELLOW("\n  No chat rooms available"))
            else:
                print(
                    f"\n{Style.Underline('NAME'):<22} {Style.Underline('ROOM ID'):<14} {Style.Underline('OWNER'):<12} {Style.Underline('PARTICIPANTS'):<15} {Style.Underline('STATUS'):<12} {Style.Underline('FEATURES')}")
                print(Style.GREY('─' * 90))

                for room in rooms:
                    name = Style.YELLOW(room['name'][:20])
                    room_id = Style.CYAN(room['room_id'])
                    owner = Style.BLUE(room['owner'][:10])
                    participants = f"{room['participants_count']}/{room['max_participants']}"

                    status_parts = []
                    if room['is_locked']:
                        status_parts.append(Style.RED('🔒 Locked'))
                    if room['is_private']:
                        status_parts.append(Style.YELLOW('🔐 Private'))
                    if not status_parts:
                        status_parts.append(Style.GREEN('🔓 Open'))
                    status = ' '.join(status_parts)[:11]

                    features = []
                    if room['voice_enabled']:
                        features.append('🎤')
                    if room['file_transfer_enabled']:
                        features.append('📁')
                    if room['is_member']:
                        features.append('✓')
                    features_str = ' '.join(features)

                    print(f"{name:<22} {room_id:<14} {owner:<12} {participants:<15} {status:<12} {features_str}")
        else:
            print(f"{Style.RED2('❌ Failed:')} {result.info}")

        input(f"\n{Style.GREY('Press Enter to continue...')}")

    def _interactive_chat(self):
        """Start interactive chat mode."""
        if not self.current_chat_room:
            print(f"{Style.RED2('❌ No active chat room. Join a room first.')}")
            input(f"\n{Style.GREY('Press Enter to continue...')}")
            return

        room = self.chat_manager.rooms.get(self.current_chat_room)
        if not room:
            print(f"{Style.RED2('❌ Room not found')}")
            input(f"\n{Style.GREY('Press Enter to continue...')}")
            return

        self.clear_screen()
        print(f"""
    {Style.CYAN('╔══════════════════════════════════════════════════════════════════════╗')}
    {Style.CYAN('║')} {Style.Bold(Style.WHITE('💬 Interactive Chat'))} - {Style.YELLOW(room.name[:30])} {' ' * (45 - len(room.name[:30]))} {Style.CYAN('║')}
    {Style.CYAN('║')} {Style.GREY('Room ID:')} {Style.CYAN(room.room_id)}{' ' * (59 - len(room.room_id))} {Style.CYAN('║')}
    {Style.CYAN('╚══════════════════════════════════════════════════════════════════════╝')}

    {Style.GREY('Commands:')} {Style.WHITE('/quit')} - Exit  {Style.WHITE('/file <path>')} - Send file  {Style.WHITE('/refresh')} - Reload messages
    """)
        print(Style.GREY('─' * 70))

        # Show recent messages
        result = self.chat_manager.get_messages(self.current_chat_room, self.current_chat_password, 20)
        message_count = 0
        if result.is_ok():
            messages = result.get()
            message_count = len(messages)
            for msg in messages[-10:]:
                self._display_message(msg)

        print(Style.GREY('─' * 70))

        # Start background listener for new messages
        def on_new_message(msg):
            # Clear current line and display new message
            print(f"\r{' ' * 80}\r", end='')  # Clear line
            self._display_message(msg)
            print(f"{Style.GREEN(f'{self.chat_manager.username}:')} ", end='', flush=True)

        listener = ChatListener(self.chat_manager, self.current_chat_room,
                                self.current_chat_password, on_new_message)
        listener.last_message_count = message_count
        listener.start()

        # Chat loop with non-blocking input
        try:
            while True:
                message = input(f"{Style.GREEN(f'{self.chat_manager.username}:')} ").strip()

                if not message:
                    continue

                if message == '/quit':
                    break

                elif message == '/refresh':
                    # Reload and show recent messages
                    result = self.chat_manager.get_messages(
                        self.current_chat_room,
                        self.current_chat_password,
                        20
                    )
                    if result.is_ok():
                        print(Style.GREY('─' * 70))
                        for msg in result.get()[-10:]:
                            self._display_message(msg)
                        print(Style.GREY('─' * 70))
                        listener.last_message_count = len(result.get())

                elif message.startswith('/file '):
                    file_path = Path(message[6:].strip())
                    self._send_file_inline(file_path)

                elif message == '/voice':
                    print(Style.YELLOW("Voice chat not yet implemented in interactive mode"))

                else:
                    result = self.chat_manager.send_message(
                        self.current_chat_room,
                        message,
                        self.current_chat_password
                    )

                    if result.is_ok():
                        # Display own message
                        self._display_message({
                            'sender': self.chat_manager.username,
                            'content': message,
                            'timestamp': datetime.now().strftime('%H:%M:%S'),
                            'message_type': 'text',
                            'is_own': True
                        })
                        # Update message count to prevent duplicate display
                        listener.last_message_count += 1
                    else:
                        print(f"{Style.RED('❌ Failed to send:')} {result.info}")

        except KeyboardInterrupt:
            pass
        finally:
            listener.stop()
            listener.join(timeout=1)

        print(f"\n{Style.YELLOW('👋 Exiting chat mode')}")
        time.sleep(1)

    def _display_message(self, msg: dict):
        """Display a chat message."""
        timestamp = Style.GREY(f"[{msg['timestamp']}]")

        if msg.get('message_type') == 'system':
            print(f"{timestamp} {Style.VIOLET2('⚙ ')} {Style.GREY(msg['content'])}")
        if msg.get('message_type') == 'file':
            sender_style = Style.GREEN if msg['is_own'] else Style.BLUE
            file_info = f"📁 {msg.get('file_name', 'Unknown')} ({msg.get('file_size', 0)} bytes)"
            sender = sender_style(f'{msg["sender"]}:')
            print(f"{timestamp} {sender} {Style.YELLOW(file_info)}")
        else:
            sender_style = Style.GREEN if msg['is_own'] else Style.BLUE
            sender = sender_style(f'{msg["sender"]}:')
            print(f"{timestamp} {sender} {Style.WHITE(msg['content'])}")

    def _send_file(self):
        """Send file in current room."""
        if not self.current_chat_room:
            print(f"{Style.RED2('❌ No active chat room')}")
            input(f"\n{Style.GREY('Press Enter to continue...')}")
            return

        print(f"\n{Style.Bold(Style.CYAN('Send File'))}")
        print(Style.GREY('─' * 70))

        file_path = input(f"{Style.WHITE('File path:')} ").strip()
        if not file_path:
            return

        self._send_file_inline(Path(file_path))
        input(f"\n{Style.GREY('Press Enter to continue...')}")

    def _send_file_inline(self, file_path: Path):
        """Send file (internal helper)."""
        if not file_path.exists():
            print(f"{Style.RED2('❌ File not found')}")
            return

        print(f"\n{Style.CYAN('📤 Sending file...')}")

        result = self.chat_manager.send_file(
            self.current_chat_room,
            file_path,
            self.current_chat_password
        )

        if result.is_ok():
            data = result.get()
            print(f"{Style.GREEN2('✅ File sent successfully!')}")
            print(f"   {Style.WHITE('File:')} {data['file_name']}")
            print(f"   {Style.WHITE('Size:')} {data['file_size']} bytes")
        else:
            print(f"{Style.RED2('❌ Failed:')} {result.info}")

    def _voice_chat(self):
        """Start live voice chat with speaker indication."""
        if not self.current_chat_room:
            print(f"{Style.RED2('❌ No active chat room')}")
            input(f"\n{Style.GREY('Press Enter to continue...')}")
            return

        room = self.chat_manager.rooms.get(self.current_chat_room)
        if not room or not room.voice_enabled:
            print(f"{Style.RED2('❌ Voice chat not enabled in this room')}")
            input(f"\n{Style.GREY('Press Enter to continue...')}")
            return

        if not VOICE_ENABLED:
            print(f"{Style.RED2('❌ pyaudio not installed')}")
            print(f"{Style.YELLOW('Install with:')} pip install pyaudio")
            input(f"\n{Style.GREY('Press Enter to continue...')}")
            return

        self.clear_screen()
        print(f"""
    {Style.CYAN('╔══════════════════════════════════════════════════════════════════════╗')}
    {Style.CYAN('║')} {Style.Bold(Style.WHITE('🎤 Live Voice Chat'))} - {Style.YELLOW(room.name[:30])} {' ' * (47 - len(room.name[:30]))} {Style.CYAN('║')}
    {Style.CYAN('║')} {Style.GREY('Press Ctrl+C to exit')} {' ' * 47} {Style.CYAN('║')}
    {Style.CYAN('╚══════════════════════════════════════════════════════════════════════╝')}
    """)

        try:
            # Initialize voice manager
            key = CryptoManager.generate_room_key(
                self.current_chat_room,
                self.current_chat_password
            )
            voice_mgr = VoiceChatManager(
                self.current_chat_room,
                key,
                self.chat_manager.username
            )

            # Check if we are the host or need to connect
            if self.current_chat_room in self.chat_manager.voice_server_info:
                # Connect to existing voice server
                host, port = self.chat_manager.voice_server_info[self.current_chat_room]
                print(f"{Style.CYAN('🔌 Connecting to voice server...')}")
                voice_mgr.connect_to_voice_server(host, port)
                print(f"{Style.GREEN2('✅ Connected to voice chat!')}\n")
            else:
                # Start as host
                print(f"{Style.CYAN('🎙️  Starting voice server...')}")
                port = voice_mgr.start_voice_server()
                self.chat_manager.voice_server_info[self.current_chat_room] = ('127.0.0.1', port)

                # Also connect to own server
                time.sleep(0.5)
                voice_mgr.connect_to_voice_server('127.0.0.1', port)
                print(f"{Style.GREEN2('✅ Voice server started on port:')} {port}")
                print(f"{Style.YELLOW('Share this info with participants:')}")
                print(f"   Host: 127.0.0.1 (or your public IP)")
                print(f"   Port: {port}\n")

            print(Style.GREY('─' * 70))
            print(f"{Style.WHITE('Voice Chat Active')} - {Style.GREEN('Speak into your microphone')}")
            print(Style.GREY('─' * 70))

            # Start recording thread
            record_thread = threading.Thread(
                target=voice_mgr.start_recording_stream,
                daemon=True
            )
            record_thread.start()

            # Display current speaker in real-time
            last_speaker = None
            print()  # Empty line for speaker display

            try:
                while True:
                    current_speaker = voice_mgr.get_current_speaker()

                    if current_speaker != last_speaker:
                        # Clear previous line and show new speaker
                        print(f"\r{' ' * 70}\r", end='')

                        if current_speaker:
                            if current_speaker == self.chat_manager.username:
                                print(f"\r{Style.GREEN('🎤 You are speaking...')}", end='', flush=True)
                            else:
                                print(f"\r{Style.CYAN(f'🎤 {current_speaker} is speaking...')}", end='', flush=True)
                        else:
                            print(f"\r{Style.GREY('🔇 Silence...')}", end='', flush=True)

                        last_speaker = current_speaker

                    time.sleep(0.1)  # Update display 10 times per second

            except KeyboardInterrupt:
                print(f"\n\n{Style.YELLOW('👋 Exiting voice chat...')}")

        except Exception as e:
            print(f"\n{Style.RED2('❌ Voice chat error:')} {e}")
            import traceback
            traceback.print_exc()

        finally:
            try:
                voice_mgr.cleanup()
            except:
                pass

        print(f"\n{Style.GREEN('Voice chat ended')}")
        input(f"\n{Style.GREY('Press Enter to continue...')}")

    def _lock_room(self):
        """Lock current room."""
        if not self.current_chat_room:
            print(f"{Style.RED2('❌ No active chat room')}")
            input(f"\n{Style.GREY('Press Enter to continue...')}")
            return

        result = self.chat_manager.lock_room(self.current_chat_room)

        if result.is_ok():
            print(f"\n{Style.GREEN2('✅ Room locked successfully!')}")
        else:
            print(f"\n{Style.RED2('❌ Failed:')} {result.info}")

        input(f"\n{Style.GREY('Press Enter to continue...')}")

    def _current_room_name(self):
        """Get name of current room."""
        if not self.current_chat_room:
            return None
        room = self.chat_manager.rooms.get(self.current_chat_room)
        return room.name if room else None

    def _leave_room(self, auto=False):
        """Leave current room."""
        if not self.current_chat_room:
            print(f"{Style.RED2('❌ No active chat room')}")
            input(f"\n{Style.GREY('Press Enter to continue...')}")
            return

        room = self.chat_manager.rooms.get(self.current_chat_room)
        room_name = room.name if room else "Unknown"

        confirm = input(f"\n{Style.YELLOW('⚠ Leave room')} '{room_name}'? (y/N): ").strip().lower() if not auto else 'y'
        if confirm != 'y':
            return

        result = self.chat_manager.leave_room(self.current_chat_room)

        if result.is_ok():
            print(f"\n{Style.GREEN2('✅ Left room successfully')}")
            self.current_chat_room = None
            self.current_chat_password = None
        else:
            print(f"\n{Style.RED2('❌ Failed:')} {result.info}")

        input(f"\n{Style.GREY('Press Enter to continue...')}")

    def p2p_menu(self):
        """P2P configuration menu."""
        while True:
            self.clear_screen()
            self.print_header()

            print(f"""
{Style.Bold(Style.WHITE('┌─ 🔧 P2P CONFIGURATION ───────────────────────────────────────────────┐'))}
{Style.WHITE('│')}                                                                      {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('1.')} {Style.WHITE('Start Relay Server')}  - Become a relay for P2P connections         {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('2.')} {Style.WHITE('Connect as Peer')}     - Connect to relay and other peers           {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('3.')} {Style.WHITE('Expose Local Service')} - Make local service accessible via P2P     {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('4.')} {Style.WHITE('Stop Instance')}       - Stop a running P2P instance                {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('0.')} {Style.WHITE('Back')}                - Return to main menu                        {Style.WHITE('│')}
{Style.WHITE('│')}                                                                      {Style.WHITE('│')}
{Style.Bold(Style.WHITE('└──────────────────────────────────────────────────────────────────────┘'))}
""")

            choice = input(f"\n{Style.CYAN('❯')} {Style.WHITE('Select option:')} ").strip()

            if choice == '0':
                break
            elif choice == '1':
                self._start_relay()
            elif choice == '2':
                self._connect_peer()
            elif choice == '3':
                self._expose_service()
            elif choice == '4':
                self._stop_instance()
            else:
                print(f"{Style.RED('Invalid option')}")
                time.sleep(1)

    def _start_relay(self):
        """Start relay server."""
        print(f"\n{Style.Bold(Style.CYAN('Start Relay Server'))}")
        print(Style.GREY('─' * 70))

        name = input(f"{Style.WHITE('Instance name (default: relay):')} ").strip() or "relay"
        bind = input(f"{Style.WHITE('Bind address (default: 0.0.0.0:9000):')} ").strip() or "0.0.0.0:9000"
        password = input(f"{Style.WHITE('Relay password:')} ").strip()

        if not password:
            print(f"{Style.RED2('❌ Password required')}")
            input(f"\n{Style.GREY('Press Enter to continue...')}")
            return

        # Get executable path
        executable = self._get_executable_path()
        if not executable:
            print(f"{Style.RED2('❌ Executable not found. Run')} {Style.WHITE('tb p2p build')} {Style.RED2('first')}")
            input(f"\n{Style.GREY('Press Enter to continue...')}")
            return

        # Create instance
        instance = EnhancedInstanceManager(name, self.app)
        config = {'bind_address': bind, 'password': password}

        success = instance.start(executable, 'relay', config)

        if success:
            self.instances[name] = instance

        input(f"\n{Style.GREY('Press Enter to continue...')}")

    def _connect_peer(self):
        """Connect as peer."""
        print(f"\n{Style.Bold(Style.CYAN('Connect as Peer'))}")
        print(Style.GREY('─' * 70))

        name = input(f"{Style.WHITE('Instance name:')} ").strip()
        if not name:
            return

        relay_addr = input(f"{Style.WHITE('Relay address (e.g., 127.0.0.1:9000):')} ").strip()
        relay_pass = input(f"{Style.WHITE('Relay password:')} ").strip()
        peer_id = input(f"{Style.WHITE('Your peer ID (default: instance name):')} ").strip() or name
        listen = input(f"{Style.WHITE('Listen address (default: 127.0.0.1:8000):')} ").strip() or "127.0.0.1:8000"
        target = input(f"{Style.WHITE('Target peer ID (optional):')} ").strip()

        if not all([relay_addr, relay_pass]):
            print(f"{Style.RED2('❌ Missing required fields')}")
            input(f"\n{Style.GREY('Press Enter to continue...')}")
            return

        # Optional: Link to chat room
        link_chat = input(f"{Style.WHITE('Link to chat room? (y/N):')} ").strip().lower() == 'y'
        chat_room = None

        if link_chat and self.current_chat_room:
            chat_room = self.current_chat_room

        # Get executable path
        executable = self._get_executable_path()
        if not executable:
            print(f"{Style.RED2('❌ Executable not found')}")
            input(f"\n{Style.GREY('Press Enter to continue...')}")
            return

        # Create instance
        instance = EnhancedInstanceManager(name, self.app)
        config = {
            'relay_address': relay_addr,
            'relay_password': relay_pass,
            'peer_id': peer_id,
            'listen_address': listen,
            'target_peer_id': target if target else None
        }

        success = instance.start(executable, 'peer', config, chat_room)

        if success:
            self.instances[name] = instance

        input(f"\n{Style.GREY('Press Enter to continue...')}")

    def _expose_service(self):
        """Expose local service via P2P."""
        print(f"\n{Style.Bold(Style.CYAN('Expose Local Service'))}")
        print(Style.GREY('─' * 70))

        name = input(f"{Style.WHITE('Instance name:')} ").strip()
        if not name:
            return

        relay_addr = input(f"{Style.WHITE('Relay address:')} ").strip()
        relay_pass = input(f"{Style.WHITE('Relay password:')} ").strip()
        peer_id = input(f"{Style.WHITE('Your peer ID:')} ").strip() or name
        listen = input(f"{Style.WHITE('Listen address (default: 127.0.0.1:8000):')} ").strip() or "127.0.0.1:8000"
        forward = input(f"{Style.WHITE('Forward to (local service, e.g., 127.0.0.1:3000):')} ").strip()

        if not all([relay_addr, relay_pass, forward]):
            print(f"{Style.RED2('❌ Missing required fields')}")
            input(f"\n{Style.GREY('Press Enter to continue...')}")
            return

        # Get executable path
        executable = self._get_executable_path()
        if not executable:
            print(f"{Style.RED2('❌ Executable not found')}")
            input(f"\n{Style.GREY('Press Enter to continue...')}")
            return

        # Create instance
        instance = EnhancedInstanceManager(name, self.app)
        config = {
            'relay_address': relay_addr,
            'relay_password': relay_pass,
            'peer_id': peer_id,
            'listen_address': listen,
            'forward_to_address': forward
        }

        success = instance.start(executable, 'peer', config)

        if success:
            self.instances[name] = instance

        input(f"\n{Style.GREY('Press Enter to continue...')}")

    def _stop_instance(self):
        """Stop running instance."""
        if not self.instances:
            print(f"\n{Style.YELLOW('No running instances')}")
            input(f"\n{Style.GREY('Press Enter to continue...')}")
            return

        print(f"\n{Style.Bold(Style.CYAN('Stop Instance'))}")
        print(Style.GREY('─' * 70))

        print(f"\n{Style.WHITE('Running instances:')}")
        running = {name: inst for name, inst in self.instances.items() if inst.is_running()}

        if not running:
            print(Style.YELLOW("  No running instances"))
            input(f"\n{Style.GREY('Press Enter to continue...')}")
            return

        for i, (name, inst) in enumerate(running.items(), 1):
            state = inst.read_state()
            mode = state.get('mode', 'Unknown')
            pid = state.get('pid', 'N/A')
            print(f"  {i}. {Style.YELLOW(name)} ({mode}, PID: {pid})")

        name = input(f"\n{Style.WHITE('Instance name to stop:')} ").strip()

        if name in running:
            running[name].stop()
        else:
            print(f"{Style.RED2('❌ Instance not found')}")

        input(f"\n{Style.GREY('Press Enter to continue...')}")

    def _get_executable_path(self) -> Optional[Path]:
        """Get executable path."""
        search_paths = [
            tb_root_dir / "bin" / EXECUTABLE_NAME,
            tb_root_dir / "tcm" / "target" / "release" / EXECUTABLE_NAME,
        ]

        for path in search_paths:
            if path.is_file():
                return path.resolve()

        return None

    def status_menu(self, do_clear=True):
        """Status and monitoring menu."""
        self.clear_screen() if do_clear else None
        self.print_header() if do_clear else None

        print(f"\n{Style.Bold(Style.CYAN('📊 System Status'))}")
        print(Style.GREY('═' * 90))

        # P2P Instances
        print(f"\n{Style.Bold(Style.WHITE('P2P Instances:'))}")
        if not self.instances:
            print(Style.YELLOW("  No instances configured"))
        else:
            print(
                f"\n{Style.Underline('NAME'):<20} {Style.Underline('MODE'):<12} {Style.Underline('STATUS'):<12} {Style.Underline('PID'):<10} {Style.Underline('CHAT ROOM')}")
            print(Style.GREY('─' * 90))

            for name, inst in self.instances.items():
                state = inst.read_state()
                mode = state.get('mode', 'Unknown')
                pid = state.get('pid', 'N/A')
                chat_room = state.get('chat_room', '-')
                status = Style.GREEN('✅ Running') if inst.is_running() else Style.RED('❌ Stopped')

                print(
                    f"{Style.YELLOW(name):<20} {mode:<12} {status:<12} {str(pid):<10} {Style.CYAN(str(chat_room)[:20])}")

        # Chat Rooms
        print(f"\n{Style.Bold(Style.WHITE('Chat Rooms:'))}")
        result = self.chat_manager.list_rooms()

        if result.is_ok():
            rooms = result.get()
            if not rooms:
                print(Style.YELLOW("  No chat rooms"))
            else:
                print(
                    f"\n{Style.Underline('NAME'):<20} {Style.Underline('PARTICIPANTS'):<15} {Style.Underline('STATUS'):<15} {Style.Underline('FEATURES')}")
                print(Style.GREY('─' * 70))

                for room in rooms:
                    name = Style.YELLOW(room['name'][:18])
                    participants = f"{room['participants_count']}/{room['max_participants']}"

                    status_parts = []
                    if room['is_locked']:
                        status_parts.append('🔒')
                    if room['is_private']:
                        status_parts.append('🔐')
                    if room['is_member']:
                        status_parts.append('✓')
                    status = ' '.join(status_parts) if status_parts else '🔓'

                    features = []
                    if room['voice_enabled']:
                        features.append('🎤 Voice')
                    if room['file_transfer_enabled']:
                        features.append('📁 Files')
                    features_str = ', '.join(features)

                    print(f"{name:<20} {participants:<15} {status:<15} {features_str}")

        input(f"\n{Style.GREY('Press Enter to continue...')}") if do_clear else None

    def settings_menu(self):
        """Settings menu."""
        while True:
            self.clear_screen()
            self.print_header()

            print(f"""
{Style.Bold(Style.WHITE('┌─ ⚙️  SETTINGS ───────────────────────────────────────────────────────┐'))}
{Style.WHITE('│')}                                                                      {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('1.')} {Style.WHITE('Change Username')}    - Set display name for chat                   {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('2.')} {Style.WHITE('Build P2P Binary')}   - Compile Rust P2P application                {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('3.')} {Style.WHITE('Clean Up')}           - Remove old instances and data               {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('0.')} {Style.WHITE('Back')}               - Return to main menu                         {Style.WHITE('│')}
{Style.WHITE('│')}                                                                      {Style.WHITE('│')}
{Style.Bold(Style.WHITE('└──────────────────────────────────────────────────────────────────────┘'))}
""")

            choice = input(f"\n{Style.CYAN('❯')} {Style.WHITE('Select option:')} ").strip()

            if choice == '0':
                break
            elif choice == '1':
                self._change_username()
            elif choice == '2':
                self._build_binary()
            elif choice == '3':
                self._cleanup()
            else:
                print(f"{Style.RED('Invalid option')}")
                time.sleep(1)

    def _change_username(self):
        """Change username."""
        print(f"\n{Style.Bold(Style.CYAN('Change Username'))}")
        print(Style.GREY('─' * 70))
        print(f"{Style.WHITE('Current:')} {Style.YELLOW(self.chat_manager.username)}")

        new_name = input(f"\n{Style.WHITE('New username:')} ").strip()
        if new_name:
            self.chat_manager.username = new_name
            print(f"\n{Style.GREEN2('✅ Username changed to:')} {Style.YELLOW(new_name)}")

        input(f"\n{Style.GREY('Press Enter to continue...')}")

    def _build_binary(self):
        """Build P2P binary."""
        print(f"\n{Style.Bold(Style.CYAN('Building P2P Binary'))}")
        print(Style.GREY('─' * 70))

        tcm_dir = tb_root_dir / "tcm"
        if not tcm_dir.exists():
            print(f"{Style.RED2('❌ TCM directory not found at:')} {tcm_dir}")
            input(f"\n{Style.GREY('Press Enter to continue...')}")
            return

        print(f"\n{Style.CYAN('⚙ Building with Cargo...')}")

        try:
            with Spinner("Compiling Rust project", symbols="t", time_in_s=120):
                process = subprocess.run(
                    ["cargo", "build", "--release"],
                    cwd=str(tcm_dir),
                    capture_output=True,
                    text=True
                )

            if process.returncode == 0:
                print(f"\n{Style.GREEN2('✅ Build successful!')}")

                # Copy to bin directory
                source = tcm_dir / "target" / "release" / EXECUTABLE_NAME
                dest_dir = tb_root_dir / "bin"
                dest_dir.mkdir(exist_ok=True)
                dest = dest_dir / EXECUTABLE_NAME

                if source.exists():
                    import shutil
                    shutil.copy2(source, dest)
                    print(f"{Style.GREEN('📦 Copied to:')} {dest}")
            else:
                print(f"\n{Style.RED2('❌ Build failed:')}")
                print(Style.GREY(process.stderr))

        except FileNotFoundError:
            print(f"\n{Style.RED2('❌ Cargo not found. Is Rust installed?')}")
        except Exception as e:
            print(f"\n{Style.RED2('❌ Build error:')} {e}")

        input(f"\n{Style.GREY('Press Enter to continue...')}")

    def _cleanup(self):
        """Cleanup old data."""
        print(f"\n{Style.Bold(Style.YELLOW('⚠ Cleanup'))}")
        print(Style.GREY('─' * 70))
        print(f"{Style.RED('This will:')}")
        print(f"  • Stop all running instances")
        print(f"  • Delete instance configurations")
        print(f"  • Keep chat rooms and messages")

        confirm = input(f"\n{Style.WHITE('Continue? (y/N):')} ").strip().lower()
        if confirm != 'y':
            return

        # Stop all instances
        for inst in self.instances.values():
            if inst.is_running():
                inst.stop()

        # Remove instance directory
        if INSTANCES_ROOT_DIR.exists():
            import shutil
            shutil.rmtree(INSTANCES_ROOT_DIR)

        self.instances = {}

        print(f"\n{Style.GREEN2('✅ Cleanup complete')}")
        input(f"\n{Style.GREY('Press Enter to continue...')}")

    def run(self):
        """Main application loop."""
        while self.running:
            self.clear_screen()
            self.print_header()
            self.print_menu()

            choice = input(f"\n{Style.CYAN('❯')} {Style.WHITE('Select option:')} ").strip()

            if choice == '0':
                print(f"\n{Style.YELLOW('👋 Goodbye!')}")
                self.running = False
            elif choice == '1':
                self.chat_menu()
            elif choice == '2':
                self.p2p_menu()
            elif choice == '3':
                self.status_menu()
            elif choice == '4':
                self.settings_menu()
            else:
                print(f"{Style.RED('Invalid option')}")
                time.sleep(1)

chat_menu()

Interactive chat menu.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py

def chat_menu(self):
    """Interactive chat menu."""
    try:
        while True:
            self.clear_screen()
            self.print_header()

            # Show current room info
            if self.current_chat_room:
                room = self.chat_manager.rooms.get(self.current_chat_room)
                if room:
                    print(f"""
{Style.GREEN('╔══ Current Room ════════════════════════════════════════════════════╗')}
{Style.GREEN('║')} {Style.WHITE('Name:')} {Style.YELLOW(room.name):<30} {Style.WHITE('ID:')} {Style.CYAN(room.room_id):<15} {" "*22+Style.GREEN('║')}
{Style.GREEN('║')} {Style.WHITE('Participants:')} {', '.join(list(room.participants)[:10]):<50}{'...' if len(room.participants) > 3 else '':<30}
{Style.GREEN('╚════════════════════════════════════════════════════════════════════╝')}
""")

            print(f"""
{Style.Bold(Style.WHITE('┌─ 💬 CHAT MENU ───────────────────────────────────────────────────────┐'))}
{Style.WHITE('│')}                                                                      {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('1.')} {Style.WHITE('Create Room')}         - Create new E2E encrypted chat room         {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('2.')} {Style.WHITE('Join Room')}           - Join existing room by ID                   {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('3.')} {Style.WHITE('List Rooms')}          - Show available chat rooms                  {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('4.')} {Style.WHITE('Interactive Chat')}    - Start live chat (current room)             {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('5.')} {Style.WHITE('Send File')}           - Transfer file (E2E encrypted)              {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('6.')} {Style.WHITE('Voice Chat')}          - Start voice chat (beta)                    {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('7.')} {Style.WHITE('Lock Room')}           - Lock current room (owner only)             {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('8.')} {Style.WHITE('Leave Room')}          - Leave current chat room                    {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('0.')} {Style.WHITE('Back')}                - Return to main menu                        {Style.WHITE('│')}
{Style.WHITE('│')}                                                                      {Style.WHITE('│')}
{Style.Bold(Style.WHITE('└──────────────────────────────────────────────────────────────────────┘'))}
""")

            choice = input(f"\n{Style.CYAN('❯')} {Style.WHITE('Select option:')} ").strip()

            if choice == '0':
                break
            elif choice == '1':
                self._create_chat_room()
            elif choice == '2':
                self._join_chat_room()
            elif choice == '3':
                self._list_chat_rooms()
            elif choice == '4':
                self._interactive_chat()
            elif choice == '5':
                self._send_file()
            elif choice == '6':
                self._voice_chat()
            elif choice == '7':
                self._lock_room()
            elif choice == '8':
                self._leave_room()
            else:
                print(f"{Style.RED('Invalid option')}")
                time.sleep(1)
    finally:
        if self._current_room_name() is not None:
            self._leave_room(auto=True)

clear_screen()

Clear terminal screen.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py

def clear_screen(self):
    """Clear terminal screen."""
    os.system('cls' if os.name == 'nt' else 'clear')

p2p_menu()

P2P configuration menu.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py

    def p2p_menu(self):
        """P2P configuration menu."""
        while True:
            self.clear_screen()
            self.print_header()

            print(f"""
{Style.Bold(Style.WHITE('┌─ 🔧 P2P CONFIGURATION ───────────────────────────────────────────────┐'))}
{Style.WHITE('│')}                                                                      {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('1.')} {Style.WHITE('Start Relay Server')}  - Become a relay for P2P connections         {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('2.')} {Style.WHITE('Connect as Peer')}     - Connect to relay and other peers           {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('3.')} {Style.WHITE('Expose Local Service')} - Make local service accessible via P2P     {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('4.')} {Style.WHITE('Stop Instance')}       - Stop a running P2P instance                {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('0.')} {Style.WHITE('Back')}                - Return to main menu                        {Style.WHITE('│')}
{Style.WHITE('│')}                                                                      {Style.WHITE('│')}
{Style.Bold(Style.WHITE('└──────────────────────────────────────────────────────────────────────┘'))}
""")

            choice = input(f"\n{Style.CYAN('❯')} {Style.WHITE('Select option:')} ").strip()

            if choice == '0':
                break
            elif choice == '1':
                self._start_relay()
            elif choice == '2':
                self._connect_peer()
            elif choice == '3':
                self._expose_service()
            elif choice == '4':
                self._stop_instance()
            else:
                print(f"{Style.RED('Invalid option')}")
                time.sleep(1)

print_header()

Print main header.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py

    def print_header(self):
        """Print main header."""
        print(f"""
{Style.CYAN('╔══════════════════════════════════════════════════════════════════════╗')}
{Style.CYAN('║')} {Style.Bold(Style.WHITE('🌐 ToolBox P2P Manager'))} {Style.CYAN('v2.0')} {Style.GREY('- Interactive Mode')} {self._current_room_name() or '':<21} {Style.CYAN('║')}
{Style.CYAN('║')} {Style.GREY('E2E Encrypted Chat • File Transfer • Voice Chat • P2P Tunnels')} {' ' * 6} {Style.CYAN('║')}
{Style.CYAN('╚══════════════════════════════════════════════════════════════════════╝')}
""")

print_menu()

Print main menu.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py

    def print_menu(self):
        """Print main menu."""
        print(f"""
{Style.Bold(Style.WHITE('┌─ 🎯 MAIN MENU ───────────────────────────────────────────────────────┐'))}
{Style.WHITE('│')}                                                                      {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('1.')} {Style.WHITE('💬 Chat Mode')}          - Start interactive E2E encrypted chat     {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('2.')} {Style.WHITE('🔧 P2P Configuration')}  - Configure P2P connections                {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('3.')} {Style.WHITE('📊 Status & Monitoring')} - View connections and rooms              {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('4.')} {Style.WHITE('⚙️  Settings')}           - Manage configuration                    {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('0.')} {Style.WHITE('🚪 Exit')}               - Quit application                         {Style.WHITE('│')}
{Style.WHITE('│')}                                                                      {Style.WHITE('│')}
{Style.Bold(Style.WHITE('└──────────────────────────────────────────────────────────────────────┘'))}
""")

run()

Main application loop.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py

def run(self):
    """Main application loop."""
    while self.running:
        self.clear_screen()
        self.print_header()
        self.print_menu()

        choice = input(f"\n{Style.CYAN('❯')} {Style.WHITE('Select option:')} ").strip()

        if choice == '0':
            print(f"\n{Style.YELLOW('👋 Goodbye!')}")
            self.running = False
        elif choice == '1':
            self.chat_menu()
        elif choice == '2':
            self.p2p_menu()
        elif choice == '3':
            self.status_menu()
        elif choice == '4':
            self.settings_menu()
        else:
            print(f"{Style.RED('Invalid option')}")
            time.sleep(1)

settings_menu()

Settings menu.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py

    def settings_menu(self):
        """Settings menu."""
        while True:
            self.clear_screen()
            self.print_header()

            print(f"""
{Style.Bold(Style.WHITE('┌─ ⚙️  SETTINGS ───────────────────────────────────────────────────────┐'))}
{Style.WHITE('│')}                                                                      {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('1.')} {Style.WHITE('Change Username')}    - Set display name for chat                   {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('2.')} {Style.WHITE('Build P2P Binary')}   - Compile Rust P2P application                {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('3.')} {Style.WHITE('Clean Up')}           - Remove old instances and data               {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('0.')} {Style.WHITE('Back')}               - Return to main menu                         {Style.WHITE('│')}
{Style.WHITE('│')}                                                                      {Style.WHITE('│')}
{Style.Bold(Style.WHITE('└──────────────────────────────────────────────────────────────────────┘'))}
""")

            choice = input(f"\n{Style.CYAN('❯')} {Style.WHITE('Select option:')} ").strip()

            if choice == '0':
                break
            elif choice == '1':
                self._change_username()
            elif choice == '2':
                self._build_binary()
            elif choice == '3':
                self._cleanup()
            else:
                print(f"{Style.RED('Invalid option')}")
                time.sleep(1)

status_menu(do_clear=True)

Status and monitoring menu.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py

def status_menu(self, do_clear=True):
    """Status and monitoring menu."""
    self.clear_screen() if do_clear else None
    self.print_header() if do_clear else None

    print(f"\n{Style.Bold(Style.CYAN('📊 System Status'))}")
    print(Style.GREY('═' * 90))

    # P2P Instances
    print(f"\n{Style.Bold(Style.WHITE('P2P Instances:'))}")
    if not self.instances:
        print(Style.YELLOW("  No instances configured"))
    else:
        print(
            f"\n{Style.Underline('NAME'):<20} {Style.Underline('MODE'):<12} {Style.Underline('STATUS'):<12} {Style.Underline('PID'):<10} {Style.Underline('CHAT ROOM')}")
        print(Style.GREY('─' * 90))

        for name, inst in self.instances.items():
            state = inst.read_state()
            mode = state.get('mode', 'Unknown')
            pid = state.get('pid', 'N/A')
            chat_room = state.get('chat_room', '-')
            status = Style.GREEN('✅ Running') if inst.is_running() else Style.RED('❌ Stopped')

            print(
                f"{Style.YELLOW(name):<20} {mode:<12} {status:<12} {str(pid):<10} {Style.CYAN(str(chat_room)[:20])}")

    # Chat Rooms
    print(f"\n{Style.Bold(Style.WHITE('Chat Rooms:'))}")
    result = self.chat_manager.list_rooms()

    if result.is_ok():
        rooms = result.get()
        if not rooms:
            print(Style.YELLOW("  No chat rooms"))
        else:
            print(
                f"\n{Style.Underline('NAME'):<20} {Style.Underline('PARTICIPANTS'):<15} {Style.Underline('STATUS'):<15} {Style.Underline('FEATURES')}")
            print(Style.GREY('─' * 70))

            for room in rooms:
                name = Style.YELLOW(room['name'][:18])
                participants = f"{room['participants_count']}/{room['max_participants']}"

                status_parts = []
                if room['is_locked']:
                    status_parts.append('🔒')
                if room['is_private']:
                    status_parts.append('🔐')
                if room['is_member']:
                    status_parts.append('✓')
                status = ' '.join(status_parts) if status_parts else '🔓'

                features = []
                if room['voice_enabled']:
                    features.append('🎤 Voice')
                if room['file_transfer_enabled']:
                    features.append('📁 Files')
                features_str = ', '.join(features)

                print(f"{name:<20} {participants:<15} {status:<15} {features_str}")

    input(f"\n{Style.GREY('Press Enter to continue...')}") if do_clear else None

P2PChatManager

Manages E2E encrypted chat rooms with file and voice support.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py

class P2PChatManager:
    """Manages E2E encrypted chat rooms with file and voice support."""

    def __init__(self, app: App, voice_server_info: Dict[str, Tuple[str, int]]):
        self.app = app
        self.rooms: Dict[str, ChatRoom] = {}
        self.current_room: Optional[str] = None
        self.username = app.get_username() or "anonymous"
        CHAT_ROOMS_DIR.mkdir(parents=True, exist_ok=True)
        self._load_rooms()
        self.file_managers: Dict[str, FileTransferManager] = {}
        self.voice_manager: Optional[VoiceChatManager] = None
        self.voice_server_info = voice_server_info

    def create_room(self, name: str, password: str, max_participants: int = 10,
                    voice_enabled: bool = False, private: bool = False) -> Result:
        """Create a new chat room."""
        room_id = hashlib.sha256(f"{name}_{self.username}_{time.time()}".encode()).hexdigest()[:12]
        encryption_key = CryptoManager.generate_room_key(room_id, password)

        room = ChatRoom(
            room_id=room_id,
            name=name,
            owner=self.username,
            participants={self.username},
            is_locked=False,
            is_private=private,
            created_at=datetime.now(),
            encryption_key=encryption_key.decode(),
            max_participants=max_participants,
            voice_enabled=voice_enabled,
            file_transfer_enabled=True
        )

        self.rooms[room_id] = room
        self._save_room(room)
        # Start voice server if voice enabled
        if voice_enabled:
            try:
                voice_mgr = VoiceChatManager(room_id, encryption_key, self.username)
                port = voice_mgr.start_voice_server()
                self.voice_server_info[room_id] = ('127.0.0.1', port)
                print(f"   {Style.GREEN('Voice server started on port:')} {port}")
            except Exception as e:
                print(f"   {Style.YELLOW(f'Warning: Could not start voice server: {e}')}")
        # Send system message
        self._send_system_message(room_id, f"Room '{name}' created by {self.username}")

        return Result.ok(data={
            'room_id': room_id,
            'name': name,
            'message': f'Room "{name}" created successfully'
        })

    def join_room(self, room_id: str, password: str) -> Result:
        """Join an existing chat room."""
        if room_id not in self.rooms:
            return Result.default_user_error("Room not found")

        room = self.rooms[room_id]

        if room.is_locked:
            return Result.default_user_error("Room is locked")

        if len(room.participants) >= room.max_participants:
            return Result.default_user_error("Room is full")

        # Verify password
        try:
            key = CryptoManager.generate_room_key(room_id, password)
            if key.decode() != room.encryption_key:
                return Result.default_user_error("Invalid password")
        except Exception:
            return Result.default_user_error("Invalid password")

        room.participants.add(self.username)
        self.current_room = room_id
        self._save_room(room)

        # Initialize file manager
        self.file_managers[room_id] = FileTransferManager(room_id, key)

        # Initialize voice manager if enabled
        # Get voice server info if available
        if room.voice_enabled:
            # Ask for voice server details if not already known
            if room_id not in self.voice_server_info:
                print(f"\n{Style.CYAN('Voice chat is enabled. Enter server details:')}")
                voice_host = input(
                    f"  {Style.WHITE('Voice server host (default: 127.0.0.1):')} ").strip() or "127.0.0.1"
                voice_port = input(f"  {Style.WHITE('Voice server port:')} ").strip()

                if voice_port and voice_port.isdigit():
                    self.voice_server_info[room_id] = (voice_host, int(voice_port))

        # Send system message
        self._send_system_message(room_id, f"{self.username} joined the room")

        return Result.ok(data={
            'room_id': room_id,
            'name': room.name,
            'participants': list(room.participants),
            'voice_enabled': room.voice_enabled,
            'file_transfer_enabled': room.file_transfer_enabled
        })

    def leave_room(self, room_id: str) -> Result:
        """Leave a chat room."""
        if room_id not in self.rooms:
            return Result.default_user_error("Room not found")

        room = self.rooms[room_id]
        if self.username not in room.participants:
            return Result.default_user_error("You are not in this room")

        # Send system message before leaving
        self._send_system_message(room_id, f"{self.username} left the room")

        room.participants.remove(self.username)

        # If owner leaves, transfer ownership or delete room
        if room.owner == self.username:
            if len(room.participants) > 0:
                room.owner = list(room.participants)[0]
                self._send_system_message(room_id, f"Room ownership transferred to {room.owner}")
            else:
                # Delete empty room
                self._delete_room(room_id)
                return Result.ok(data="Room deleted (no participants)")

        self._save_room(room)

        if self.current_room == room_id:
            self.current_room = None

        # Cleanup managers
        if room_id in self.file_managers:
            del self.file_managers[room_id]
        if self.voice_manager:
            self.voice_manager.cleanup()
            self.voice_manager = None

        return Result.ok(data="Left room successfully")

    def lock_room(self, room_id: str) -> Result:
        """Lock a room to prevent new participants."""
        if room_id not in self.rooms:
            return Result.default_user_error("Room not found")

        room = self.rooms[room_id]

        if room.owner != self.username:
            return Result.default_user_error("Only room owner can lock the room")

        room.is_locked = True
        room.is_private = True
        self._save_room(room)

        self._send_system_message(room_id, f"Room locked by {self.username}")

        return Result.ok(data=f'Room "{room.name}" is now locked and private')

    def send_message(self, room_id: str, content: str, password: str) -> Result:
        """Send encrypted text message to room."""
        if room_id not in self.rooms:
            return Result.default_user_error("Room not found")

        room = self.rooms[room_id]
        if self.username not in room.participants:
            return Result.default_user_error("You are not in this room")

        try:
            key = CryptoManager.generate_room_key(room_id, password)
            encrypted_content = CryptoManager.encrypt_message(content, key)

            message = ChatMessage(
                sender=self.username,
                content=encrypted_content,
                timestamp=datetime.now(),
                room_id=room_id,
                message_type=MessageType.TEXT,
                encrypted=True
            )

            self._save_message(message)
            return Result.ok(data="Message sent")

        except Exception as e:
            return Result.default_internal_error(f"Failed to send message: {e}")

    def send_file(self, room_id: str, file_path: Path, password: str) -> Result:
        """Send encrypted file to room."""
        if room_id not in self.rooms:
            return Result.default_user_error("Room not found")

        room = self.rooms[room_id]
        if not room.file_transfer_enabled:
            return Result.default_user_error("File transfer disabled in this room")

        if self.username not in room.participants:
            return Result.default_user_error("You are not in this room")

        try:
            # Prepare file for transfer
            file_manager = self.file_managers.get(room_id)
            if not file_manager:
                key = CryptoManager.generate_room_key(room_id, password)
                file_manager = FileTransferManager(room_id, key)
                self.file_managers[room_id] = file_manager

            transfer_id, file_size = file_manager.prepare_file(file_path)

            # Create file message
            key = CryptoManager.generate_room_key(room_id, password)
            encrypted_content = CryptoManager.encrypt_message(transfer_id, key)

            message = ChatMessage(
                sender=self.username,
                content=encrypted_content,
                timestamp=datetime.now(),
                room_id=room_id,
                message_type=MessageType.FILE,
                encrypted=True,
                file_name=file_path.name,
                file_size=file_size
            )

            self._save_message(message)

            return Result.ok(data={
                'transfer_id': transfer_id,
                'file_name': file_path.name,
                'file_size': file_size
            })

        except Exception as e:
            return Result.default_internal_error(f"Failed to send file: {e}")

    def receive_file(self, room_id: str, transfer_id: str, file_name: str) -> Result:
        """Receive and decrypt file from room."""
        if room_id not in self.rooms:
            return Result.default_user_error("Room not found")

        try:
            file_manager = self.file_managers.get(room_id)
            if not file_manager:
                return Result.default_user_error("File manager not initialized")

            output_path = file_manager.receive_file(transfer_id, file_name)

            return Result.ok(data={
                'file_path': str(output_path),
                'file_name': file_name
            })

        except Exception as e:
            return Result.default_internal_error(f"Failed to receive file: {e}")

    def get_messages(self, room_id: str, password: str, limit: int = 50) -> Result:
        """Get decrypted messages from room."""
        if room_id not in self.rooms:
            return Result.default_user_error("Room not found")

        room = self.rooms[room_id]
        if self.username not in room.participants:
            return Result.default_user_error("You are not in this room")

        try:
            key = CryptoManager.generate_room_key(room_id, password)
            messages = self._load_messages(room_id, limit)

            decrypted_messages = []
            for msg in messages:
                if msg.encrypted and msg.message_type != MessageType.SYSTEM:
                    try:
                        decrypted_content = CryptoManager.decrypt_message(msg.content, key)
                        decrypted_messages.append({
                            'sender': msg.sender,
                            'content': decrypted_content,
                            'timestamp': msg.timestamp.strftime('%H:%M:%S'),
                            'message_type': msg.message_type.value,
                            'is_own': msg.sender == self.username,
                            'file_name': msg.file_name,
                            'file_size': msg.file_size
                        })
                    except Exception:
                        continue
                else:
                    decrypted_messages.append({
                        'sender': msg.sender,
                        'content': msg.content,
                        'timestamp': msg.timestamp.strftime('%H:%M:%S'),
                        'message_type': msg.message_type.value,
                        'is_own': False
                    })

            return Result.ok(data=decrypted_messages)

        except Exception as e:
            return Result.default_internal_error(f"Failed to get messages: {e}")

    def list_rooms(self, show_all: bool = False) -> Result:
        """List available rooms for user."""
        user_rooms = []
        for room in self.rooms.values():
            # Show only user's rooms unless show_all is True
            if show_all or self.username in room.participants:
                # Don't show private/locked rooms to non-participants
                if room.is_private and self.username not in room.participants:
                    continue

                user_rooms.append({
                    'room_id': room.room_id,
                    'name': room.name,
                    'owner': room.owner,
                    'participants_count': len(room.participants),
                    'max_participants': room.max_participants,
                    'is_locked': room.is_locked,
                    'is_private': room.is_private,
                    'voice_enabled': room.voice_enabled,
                    'file_transfer_enabled': room.file_transfer_enabled,
                    'created_at': room.created_at.strftime('%Y-%m-%d %H:%M'),
                    'is_member': self.username in room.participants
                })

        return Result.ok(data=user_rooms)

    def _send_system_message(self, room_id: str, content: str):
        """Send a system message (not encrypted)."""
        message = ChatMessage(
            sender="SYSTEM",
            content=content,
            timestamp=datetime.now(),
            room_id=room_id,
            message_type=MessageType.SYSTEM,
            encrypted=False
        )
        self._save_message(message)

    def _save_room(self, room: ChatRoom):
        """Save room to storage."""
        room_file = CHAT_ROOMS_DIR / f"room_{room.room_id}.json"

        with open(room_file, 'w') as f:
            json.dump(room.to_dict(), f, indent=2)

    def _load_rooms(self):
        """Load rooms from storage."""
        if not CHAT_ROOMS_DIR.exists():
            return

        for room_file in CHAT_ROOMS_DIR.glob("room_*.json"):
            try:
                with open(room_file) as f:
                    room_data = json.load(f)
                    room = ChatRoom.from_dict(room_data)
                    self.rooms[room.room_id] = room
            except Exception as e:
                print(f"Warning: Failed to load room {room_file}: {e}")

    def _delete_room(self, room_id: str):
        """Delete a room and its messages."""
        if room_id in self.rooms:
            del self.rooms[room_id]

        # Delete room file
        room_file = CHAT_ROOMS_DIR / f"room_{room_id}.json"
        if room_file.exists():
            room_file.unlink()

        # Delete messages file
        messages_file = CHAT_ROOMS_DIR / f"messages_{room_id}.jsonl"
        if messages_file.exists():
            messages_file.unlink()

    def _save_message(self, message: ChatMessage):
        """Save message to storage."""
        messages_file = CHAT_ROOMS_DIR / f"messages_{message.room_id}.jsonl"
        with open(messages_file, 'a') as f:
            f.write(json.dumps(message.to_dict()) + '\n')

    def _load_messages(self, room_id: str, limit: int = 50) -> List[ChatMessage]:
        """Load messages from storage."""
        messages_file = CHAT_ROOMS_DIR / f"messages_{room_id}.jsonl"
        if not messages_file.exists():
            return []

        messages = []
        with open(messages_file) as f:
            lines = f.readlines()
            for line in lines[-limit:]:
                try:
                    message_data = json.loads(line.strip())
                    messages.append(ChatMessage.from_dict(message_data))
                except Exception:
                    continue

        return messages

create_room(name, password, max_participants=10, voice_enabled=False, private=False)

Create a new chat room.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py

def create_room(self, name: str, password: str, max_participants: int = 10,
                voice_enabled: bool = False, private: bool = False) -> Result:
    """Create a new chat room."""
    room_id = hashlib.sha256(f"{name}_{self.username}_{time.time()}".encode()).hexdigest()[:12]
    encryption_key = CryptoManager.generate_room_key(room_id, password)

    room = ChatRoom(
        room_id=room_id,
        name=name,
        owner=self.username,
        participants={self.username},
        is_locked=False,
        is_private=private,
        created_at=datetime.now(),
        encryption_key=encryption_key.decode(),
        max_participants=max_participants,
        voice_enabled=voice_enabled,
        file_transfer_enabled=True
    )

    self.rooms[room_id] = room
    self._save_room(room)
    # Start voice server if voice enabled
    if voice_enabled:
        try:
            voice_mgr = VoiceChatManager(room_id, encryption_key, self.username)
            port = voice_mgr.start_voice_server()
            self.voice_server_info[room_id] = ('127.0.0.1', port)
            print(f"   {Style.GREEN('Voice server started on port:')} {port}")
        except Exception as e:
            print(f"   {Style.YELLOW(f'Warning: Could not start voice server: {e}')}")
    # Send system message
    self._send_system_message(room_id, f"Room '{name}' created by {self.username}")

    return Result.ok(data={
        'room_id': room_id,
        'name': name,
        'message': f'Room "{name}" created successfully'
    })

get_messages(room_id, password, limit=50)

Get decrypted messages from room.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py

def get_messages(self, room_id: str, password: str, limit: int = 50) -> Result:
    """Get decrypted messages from room."""
    if room_id not in self.rooms:
        return Result.default_user_error("Room not found")

    room = self.rooms[room_id]
    if self.username not in room.participants:
        return Result.default_user_error("You are not in this room")

    try:
        key = CryptoManager.generate_room_key(room_id, password)
        messages = self._load_messages(room_id, limit)

        decrypted_messages = []
        for msg in messages:
            if msg.encrypted and msg.message_type != MessageType.SYSTEM:
                try:
                    decrypted_content = CryptoManager.decrypt_message(msg.content, key)
                    decrypted_messages.append({
                        'sender': msg.sender,
                        'content': decrypted_content,
                        'timestamp': msg.timestamp.strftime('%H:%M:%S'),
                        'message_type': msg.message_type.value,
                        'is_own': msg.sender == self.username,
                        'file_name': msg.file_name,
                        'file_size': msg.file_size
                    })
                except Exception:
                    continue
            else:
                decrypted_messages.append({
                    'sender': msg.sender,
                    'content': msg.content,
                    'timestamp': msg.timestamp.strftime('%H:%M:%S'),
                    'message_type': msg.message_type.value,
                    'is_own': False
                })

        return Result.ok(data=decrypted_messages)

    except Exception as e:
        return Result.default_internal_error(f"Failed to get messages: {e}")

join_room(room_id, password)

Join an existing chat room.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py

def join_room(self, room_id: str, password: str) -> Result:
    """Join an existing chat room."""
    if room_id not in self.rooms:
        return Result.default_user_error("Room not found")

    room = self.rooms[room_id]

    if room.is_locked:
        return Result.default_user_error("Room is locked")

    if len(room.participants) >= room.max_participants:
        return Result.default_user_error("Room is full")

    # Verify password
    try:
        key = CryptoManager.generate_room_key(room_id, password)
        if key.decode() != room.encryption_key:
            return Result.default_user_error("Invalid password")
    except Exception:
        return Result.default_user_error("Invalid password")

    room.participants.add(self.username)
    self.current_room = room_id
    self._save_room(room)

    # Initialize file manager
    self.file_managers[room_id] = FileTransferManager(room_id, key)

    # Initialize voice manager if enabled
    # Get voice server info if available
    if room.voice_enabled:
        # Ask for voice server details if not already known
        if room_id not in self.voice_server_info:
            print(f"\n{Style.CYAN('Voice chat is enabled. Enter server details:')}")
            voice_host = input(
                f"  {Style.WHITE('Voice server host (default: 127.0.0.1):')} ").strip() or "127.0.0.1"
            voice_port = input(f"  {Style.WHITE('Voice server port:')} ").strip()

            if voice_port and voice_port.isdigit():
                self.voice_server_info[room_id] = (voice_host, int(voice_port))

    # Send system message
    self._send_system_message(room_id, f"{self.username} joined the room")

    return Result.ok(data={
        'room_id': room_id,
        'name': room.name,
        'participants': list(room.participants),
        'voice_enabled': room.voice_enabled,
        'file_transfer_enabled': room.file_transfer_enabled
    })

leave_room(room_id)

Leave a chat room.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py

def leave_room(self, room_id: str) -> Result:
    """Leave a chat room."""
    if room_id not in self.rooms:
        return Result.default_user_error("Room not found")

    room = self.rooms[room_id]
    if self.username not in room.participants:
        return Result.default_user_error("You are not in this room")

    # Send system message before leaving
    self._send_system_message(room_id, f"{self.username} left the room")

    room.participants.remove(self.username)

    # If owner leaves, transfer ownership or delete room
    if room.owner == self.username:
        if len(room.participants) > 0:
            room.owner = list(room.participants)[0]
            self._send_system_message(room_id, f"Room ownership transferred to {room.owner}")
        else:
            # Delete empty room
            self._delete_room(room_id)
            return Result.ok(data="Room deleted (no participants)")

    self._save_room(room)

    if self.current_room == room_id:
        self.current_room = None

    # Cleanup managers
    if room_id in self.file_managers:
        del self.file_managers[room_id]
    if self.voice_manager:
        self.voice_manager.cleanup()
        self.voice_manager = None

    return Result.ok(data="Left room successfully")

list_rooms(show_all=False)

List available rooms for user.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py

def list_rooms(self, show_all: bool = False) -> Result:
    """List available rooms for user."""
    user_rooms = []
    for room in self.rooms.values():
        # Show only user's rooms unless show_all is True
        if show_all or self.username in room.participants:
            # Don't show private/locked rooms to non-participants
            if room.is_private and self.username not in room.participants:
                continue

            user_rooms.append({
                'room_id': room.room_id,
                'name': room.name,
                'owner': room.owner,
                'participants_count': len(room.participants),
                'max_participants': room.max_participants,
                'is_locked': room.is_locked,
                'is_private': room.is_private,
                'voice_enabled': room.voice_enabled,
                'file_transfer_enabled': room.file_transfer_enabled,
                'created_at': room.created_at.strftime('%Y-%m-%d %H:%M'),
                'is_member': self.username in room.participants
            })

    return Result.ok(data=user_rooms)

lock_room(room_id)

Lock a room to prevent new participants.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py

def lock_room(self, room_id: str) -> Result:
    """Lock a room to prevent new participants."""
    if room_id not in self.rooms:
        return Result.default_user_error("Room not found")

    room = self.rooms[room_id]

    if room.owner != self.username:
        return Result.default_user_error("Only room owner can lock the room")

    room.is_locked = True
    room.is_private = True
    self._save_room(room)

    self._send_system_message(room_id, f"Room locked by {self.username}")

    return Result.ok(data=f'Room "{room.name}" is now locked and private')

receive_file(room_id, transfer_id, file_name)

Receive and decrypt file from room.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py

def receive_file(self, room_id: str, transfer_id: str, file_name: str) -> Result:
    """Receive and decrypt file from room."""
    if room_id not in self.rooms:
        return Result.default_user_error("Room not found")

    try:
        file_manager = self.file_managers.get(room_id)
        if not file_manager:
            return Result.default_user_error("File manager not initialized")

        output_path = file_manager.receive_file(transfer_id, file_name)

        return Result.ok(data={
            'file_path': str(output_path),
            'file_name': file_name
        })

    except Exception as e:
        return Result.default_internal_error(f"Failed to receive file: {e}")

send_file(room_id, file_path, password)

Send encrypted file to room.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py

def send_file(self, room_id: str, file_path: Path, password: str) -> Result:
    """Send encrypted file to room."""
    if room_id not in self.rooms:
        return Result.default_user_error("Room not found")

    room = self.rooms[room_id]
    if not room.file_transfer_enabled:
        return Result.default_user_error("File transfer disabled in this room")

    if self.username not in room.participants:
        return Result.default_user_error("You are not in this room")

    try:
        # Prepare file for transfer
        file_manager = self.file_managers.get(room_id)
        if not file_manager:
            key = CryptoManager.generate_room_key(room_id, password)
            file_manager = FileTransferManager(room_id, key)
            self.file_managers[room_id] = file_manager

        transfer_id, file_size = file_manager.prepare_file(file_path)

        # Create file message
        key = CryptoManager.generate_room_key(room_id, password)
        encrypted_content = CryptoManager.encrypt_message(transfer_id, key)

        message = ChatMessage(
            sender=self.username,
            content=encrypted_content,
            timestamp=datetime.now(),
            room_id=room_id,
            message_type=MessageType.FILE,
            encrypted=True,
            file_name=file_path.name,
            file_size=file_size
        )

        self._save_message(message)

        return Result.ok(data={
            'transfer_id': transfer_id,
            'file_name': file_path.name,
            'file_size': file_size
        })

    except Exception as e:
        return Result.default_internal_error(f"Failed to send file: {e}")

send_message(room_id, content, password)

Send encrypted text message to room.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py

def send_message(self, room_id: str, content: str, password: str) -> Result:
    """Send encrypted text message to room."""
    if room_id not in self.rooms:
        return Result.default_user_error("Room not found")

    room = self.rooms[room_id]
    if self.username not in room.participants:
        return Result.default_user_error("You are not in this room")

    try:
        key = CryptoManager.generate_room_key(room_id, password)
        encrypted_content = CryptoManager.encrypt_message(content, key)

        message = ChatMessage(
            sender=self.username,
            content=encrypted_content,
            timestamp=datetime.now(),
            room_id=room_id,
            message_type=MessageType.TEXT,
            encrypted=True
        )

        self._save_message(message)
        return Result.ok(data="Message sent")

    except Exception as e:
        return Result.default_internal_error(f"Failed to send message: {e}")

P2PConnection dataclass

Represents a P2P connection configuration.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py

@dataclass
class P2PConnection:
    """Represents a P2P connection configuration."""
    name: str
    mode: str  # relay, peer-provider, peer-consumer
    status: str  # active, stopped, error
    pid: Optional[int] = None
    config: dict = field(default_factory=dict)
    chat_room: Optional[str] = None

VoiceChatManager

Manages P2P voice chat with live streaming and speaker detection.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py

class VoiceChatManager:
    """Manages P2P voice chat with live streaming and speaker detection."""

    def __init__(self, room_id: str, encryption_key: bytes, username: str):
        self.room_id = room_id
        self.encryption_key = encryption_key
        self.username = username
        self.is_recording = False
        self.is_playing = False
        self.current_speaker = None
        self.voice_server_port = None

        if not VOICE_ENABLED:
            return
            raise RuntimeError("pyaudio not installed. Install with: pip install pyaudio")

        self.audio = pyaudio.PyAudio()
        self.voice_dir = VOICE_CACHE_DIR / room_id
        self.voice_dir.mkdir(parents=True, exist_ok=True)

        # Voice activity detection
        self.voice_threshold = 500  # Audio level threshold
        self.speaking = False

        # Network
        self.server_socket = None
        self.clients = {}  # {addr: socket}
        self.running = False

    def calculate_rms(self, audio_data):
        """Calculate RMS (Root Mean Square) for voice activity detection."""
        import array
        count = len(audio_data) / 2
        format_str = "%dh" % count
        shorts = array.array('h', audio_data)
        sum_squares = sum((sample ** 2 for sample in shorts))
        rms = (sum_squares / count) ** 0.5
        return rms

    def start_voice_server(self, port: int = 0):
        """Start voice relay server for this room."""
        self.server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        self.server_socket.bind(('0.0.0.0', port))
        self.server_socket.listen(5)
        self.server_socket.settimeout(0.5)

        self.voice_server_port = self.server_socket.getsockname()[1]
        self.running = True

        # Start accepting clients
        accept_thread = threading.Thread(target=self._accept_clients, daemon=True)
        accept_thread.start()

        return self.voice_server_port

    def _accept_clients(self):
        """Accept incoming voice client connections."""
        while self.running:
            try:
                client_sock, addr = self.server_socket.accept()
                client_sock.settimeout(1.0)
                self.clients[addr] = client_sock
                print(f"\r{Style.GREEN('🎤 New voice participant connected')}{' ' * 20}")

                # Start receiving thread for this client
                recv_thread = threading.Thread(
                    target=self._receive_from_client,
                    args=(client_sock, addr),
                    daemon=True
                )
                recv_thread.start()

            except socket.timeout:
                continue
            except Exception as e:
                if self.running:
                    print(f"\r{Style.RED(f'Voice server error: {e}')}{' ' * 20}")

    def _receive_from_client(self, client_sock, addr):
        """Receive audio from client and broadcast to others."""
        try:
            while self.running:
                try:
                    # Receive packet size
                    size_data = client_sock.recv(4)
                    if not size_data or len(size_data) < 4:
                        break

                    packet_size = int.from_bytes(size_data, 'big')

                    # Sanity check
                    if packet_size > 1024 * 1024:  # 1MB max
                        break

                    # Receive full packet
                    packet = b''
                    while len(packet) < packet_size:
                        remaining = packet_size - len(packet)
                        chunk = client_sock.recv(min(remaining, 4096))
                        if not chunk:
                            break
                        packet += chunk

                    if len(packet) != packet_size:
                        break

                    # Parse packet header to update speaker
                    if len(packet) >= 3:
                        username_len = int.from_bytes(packet[:2], 'big')
                        if len(packet) >= 2 + username_len + 1:
                            username = packet[2:2 + username_len].decode('utf-8')
                            is_speaking = packet[2 + username_len] == 1

                            # Update current speaker
                            if is_speaking:
                                self.current_speaker = username
                            elif self.current_speaker == username:
                                self.current_speaker = None

                    # Broadcast to all other clients
                    self._broadcast_audio(packet, addr)

                except socket.timeout:
                    continue
                except Exception:
                    break

        except Exception:
            pass
        finally:
            if addr in self.clients:
                del self.clients[addr]
                print(f"\r{Style.YELLOW('Voice participant disconnected')}{' ' * 20}")
            try:
                client_sock.close()
            except:
                pass

    def _broadcast_audio(self, packet, exclude_addr):
        """Broadcast audio packet to all clients except sender."""
        dead_clients = []
        for addr, client_sock in self.clients.items():
            if addr == exclude_addr:
                continue
            try:
                # Send packet size then packet
                size_bytes = len(packet).to_bytes(4, 'big')
                client_sock.sendall(size_bytes + packet)
            except:
                dead_clients.append(addr)

        # Remove dead clients
        for addr in dead_clients:
            if addr in self.clients:
                del self.clients[addr]

    def connect_to_voice_server(self, host: str, port: int):
        """Connect to voice relay server."""
        self.client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.client_socket.connect((host, port))
        self.client_socket.settimeout(1.0)
        self.running = True

        # Start playback thread
        playback_thread = threading.Thread(target=self._playback_loop, daemon=True)
        playback_thread.start()

    def _playback_loop(self):
        """Receive and play audio from server."""
        stream = self.audio.open(
            format=VOICE_FORMAT,
            channels=VOICE_CHANNELS,
            rate=VOICE_RATE,
            output=True,
            frames_per_buffer=VOICE_CHUNK
        )

        print(f"{Style.GREEN('🔊 Playback active - Listening...')}")

        try:
            while self.running:
                try:
                    # Receive packet size
                    size_data = self.client_socket.recv(4)
                    if not size_data or len(size_data) < 4:
                        break

                    packet_size = int.from_bytes(size_data, 'big')

                    # Sanity check
                    if packet_size > 1024 * 1024:  # 1MB max
                        print(f"\r{Style.RED('Invalid packet size')}{' ' * 20}")
                        break

                    # Receive full packet
                    packet = b''
                    while len(packet) < packet_size:
                        remaining = packet_size - len(packet)
                        chunk = self.client_socket.recv(min(remaining, 4096))
                        if not chunk:
                            break
                        packet += chunk

                    if len(packet) != packet_size:
                        break

                    # Parse packet
                    if len(packet) < 3:
                        continue

                    username_len = int.from_bytes(packet[:2], 'big')
                    if len(packet) < 2 + username_len + 1:
                        continue

                    username = packet[2:2 + username_len].decode('utf-8')
                    is_speaking = packet[2 + username_len] == 1
                    audio_data = packet[2 + username_len + 1:]

                    # Update speaker
                    if is_speaking:
                        self.current_speaker = username
                    elif self.current_speaker == username:
                        self.current_speaker = None

                    # Decrypt and play if there's audio data
                    if len(audio_data) > 0:
                        try:
                            decrypted_audio = CryptoManager.decrypt_bytes(
                                audio_data,
                                self.encryption_key
                            )
                            stream.write(decrypted_audio)
                        except Exception as e:
                            # Decryption failed, skip this packet
                            pass

                except socket.timeout:
                    continue
                except Exception as e:
                    if self.running:
                        print(f"\r{Style.RED(f'Playback error: {e}')}{' ' * 20}")
                    break

        finally:
            stream.stop_stream()
            stream.close()
            print(f"\n{Style.YELLOW('🔇 Playback stopped')}")

    def start_recording_stream(self):
        """Start streaming microphone input to server."""
        self.is_recording = True

        stream = self.audio.open(
            format=VOICE_FORMAT,
            channels=VOICE_CHANNELS,
            rate=VOICE_RATE,
            input=True,
            frames_per_buffer=VOICE_CHUNK
        )

        print(f"{Style.GREEN('🎤 Microphone active - Start speaking!')}")

        try:
            silence_counter = 0
            while self.is_recording:
                try:
                    # Read audio
                    audio_data = stream.read(VOICE_CHUNK, exception_on_overflow=False)

                    # Voice activity detection
                    rms = self.calculate_rms(audio_data)
                    is_speaking = rms > self.voice_threshold

                    if is_speaking:
                        self.speaking = True
                        silence_counter = 0

                        # Encrypt audio directly as bytes
                        encrypted_bytes = CryptoManager.encrypt_bytes(
                            audio_data,
                            self.encryption_key
                        )

                        # Build packet: [username_len(2)][username][speaker_flag(1)][audio_data]
                        username_bytes = self.username.encode('utf-8')
                        username_len = len(username_bytes).to_bytes(2, 'big')
                        speaker_flag = b'\x01'

                        packet = username_len + username_bytes + speaker_flag + encrypted_bytes

                        # Send to server
                        try:
                            size_bytes = len(packet).to_bytes(4, 'big')
                            self.client_socket.sendall(size_bytes + packet)
                        except Exception as e:
                            print(f"\r{Style.RED(f'Send error: {e}')}{' ' * 20}")
                            break
                    else:
                        silence_counter += 1

                        # Send stop-speaking packet after 3 consecutive silent chunks
                        if self.speaking and silence_counter > 3:
                            username_bytes = self.username.encode('utf-8')
                            username_len = len(username_bytes).to_bytes(2, 'big')
                            packet = username_len + username_bytes + b'\x00'

                            try:
                                size_bytes = len(packet).to_bytes(4, 'big')
                                self.client_socket.sendall(size_bytes + packet)
                            except:
                                pass

                            self.speaking = False

                except Exception as e:
                    if self.is_recording:
                        print(f"\r{Style.RED(f'Recording error: {e}')}{' ' * 20}")
                    break

        finally:
            stream.stop_stream()
            stream.close()
            print(f"\n{Style.YELLOW('🔇 Microphone stopped')}")

    def stop_recording(self):
        """Stop recording stream."""
        self.is_recording = False

    def get_current_speaker(self):
        """Get username of current speaker."""
        return self.current_speaker

    def cleanup(self):
        """Cleanup voice resources."""
        self.running = False
        self.is_recording = False

        if hasattr(self, 'client_socket'):
            try:
                self.client_socket.close()
            except:
                pass

        if self.server_socket:
            try:
                self.server_socket.close()
            except:
                pass

        # Close all client connections
        for client_sock in self.clients.values():
            try:
                client_sock.close()
            except:
                pass

        self.clients.clear()

        try:
            self.audio.terminate()
        except:
            pass

calculate_rms(audio_data)

Calculate RMS (Root Mean Square) for voice activity detection.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py

def calculate_rms(self, audio_data):
    """Calculate RMS (Root Mean Square) for voice activity detection."""
    import array
    count = len(audio_data) / 2
    format_str = "%dh" % count
    shorts = array.array('h', audio_data)
    sum_squares = sum((sample ** 2 for sample in shorts))
    rms = (sum_squares / count) ** 0.5
    return rms

cleanup()

Cleanup voice resources.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py

def cleanup(self):
    """Cleanup voice resources."""
    self.running = False
    self.is_recording = False

    if hasattr(self, 'client_socket'):
        try:
            self.client_socket.close()
        except:
            pass

    if self.server_socket:
        try:
            self.server_socket.close()
        except:
            pass

    # Close all client connections
    for client_sock in self.clients.values():
        try:
            client_sock.close()
        except:
            pass

    self.clients.clear()

    try:
        self.audio.terminate()
    except:
        pass

connect_to_voice_server(host, port)

Connect to voice relay server.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py

def connect_to_voice_server(self, host: str, port: int):
    """Connect to voice relay server."""
    self.client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    self.client_socket.connect((host, port))
    self.client_socket.settimeout(1.0)
    self.running = True

    # Start playback thread
    playback_thread = threading.Thread(target=self._playback_loop, daemon=True)
    playback_thread.start()

get_current_speaker()

Get username of current speaker.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py

def get_current_speaker(self):
    """Get username of current speaker."""
    return self.current_speaker

start_recording_stream()

Start streaming microphone input to server.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py

def start_recording_stream(self):
    """Start streaming microphone input to server."""
    self.is_recording = True

    stream = self.audio.open(
        format=VOICE_FORMAT,
        channels=VOICE_CHANNELS,
        rate=VOICE_RATE,
        input=True,
        frames_per_buffer=VOICE_CHUNK
    )

    print(f"{Style.GREEN('🎤 Microphone active - Start speaking!')}")

    try:
        silence_counter = 0
        while self.is_recording:
            try:
                # Read audio
                audio_data = stream.read(VOICE_CHUNK, exception_on_overflow=False)

                # Voice activity detection
                rms = self.calculate_rms(audio_data)
                is_speaking = rms > self.voice_threshold

                if is_speaking:
                    self.speaking = True
                    silence_counter = 0

                    # Encrypt audio directly as bytes
                    encrypted_bytes = CryptoManager.encrypt_bytes(
                        audio_data,
                        self.encryption_key
                    )

                    # Build packet: [username_len(2)][username][speaker_flag(1)][audio_data]
                    username_bytes = self.username.encode('utf-8')
                    username_len = len(username_bytes).to_bytes(2, 'big')
                    speaker_flag = b'\x01'

                    packet = username_len + username_bytes + speaker_flag + encrypted_bytes

                    # Send to server
                    try:
                        size_bytes = len(packet).to_bytes(4, 'big')
                        self.client_socket.sendall(size_bytes + packet)
                    except Exception as e:
                        print(f"\r{Style.RED(f'Send error: {e}')}{' ' * 20}")
                        break
                else:
                    silence_counter += 1

                    # Send stop-speaking packet after 3 consecutive silent chunks
                    if self.speaking and silence_counter > 3:
                        username_bytes = self.username.encode('utf-8')
                        username_len = len(username_bytes).to_bytes(2, 'big')
                        packet = username_len + username_bytes + b'\x00'

                        try:
                            size_bytes = len(packet).to_bytes(4, 'big')
                            self.client_socket.sendall(size_bytes + packet)
                        except:
                            pass

                        self.speaking = False

            except Exception as e:
                if self.is_recording:
                    print(f"\r{Style.RED(f'Recording error: {e}')}{' ' * 20}")
                break

    finally:
        stream.stop_stream()
        stream.close()
        print(f"\n{Style.YELLOW('🔇 Microphone stopped')}")

start_voice_server(port=0)

Start voice relay server for this room.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py

def start_voice_server(self, port: int = 0):
    """Start voice relay server for this room."""
    self.server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    self.server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    self.server_socket.bind(('0.0.0.0', port))
    self.server_socket.listen(5)
    self.server_socket.settimeout(0.5)

    self.voice_server_port = self.server_socket.getsockname()[1]
    self.running = True

    # Start accepting clients
    accept_thread = threading.Thread(target=self._accept_clients, daemon=True)
    accept_thread.start()

    return self.voice_server_port

stop_recording()

Stop recording stream.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py

def stop_recording(self):
    """Stop recording stream."""
    self.is_recording = False

cli_tcm_runner()

Main CLI entry point.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py

def cli_tcm_runner():
    """Main CLI entry point."""
    parser = argparse.ArgumentParser(
        description=f"🚀 {Style.Bold('ToolBox P2P Manager')} - Advanced P2P with E2E Chat",
        formatter_class=argparse.RawTextHelpFormatter
    )

    parser.add_argument('--interactive', '-i', action='store_true',
                        help='Start interactive mode (default)')
    parser.add_argument("status", nargs='?', const=True,
                        help='Check status of all instances')
    args = parser.parse_args()

    # Always start in interactive mode
    cli = InteractiveP2PCLI()

    if args.status:
        cli.status_menu(do_clear=False)
        return

    try:
        cli.run()
    except KeyboardInterrupt:
        print(f"\n\n{Style.YELLOW('👋 Interrupted by user. Goodbye!')}")
    except Exception as e:
        print(f"\n{Style.RED2('❌ Fatal error:')} {e}")
        import traceback
        traceback.print_exc()
    finally:
        # Cleanup
        print(f"\n{Style.GREY('Cleaning up...')}")

user_dashboard

interactive_user_dashboard() async

Modern interactive user dashboard and mini CLI

Source code in toolboxv2/utils/clis/user_dashboard.py

async def interactive_user_dashboard():
    """Modern interactive user dashboard and mini CLI"""
    import asyncio
    from pathlib import Path

    # =================== UI Helper Functions ===================
    # Note: print_box_header, print_box_content, print_box_footer, print_status, print_separator
    # are now imported from cli_printing at the top of the file

    def get_key():
        """Get single keypress (cross-platform)"""
        if system() == "Windows":
            import msvcrt
            key = msvcrt.getch()
            if key == b'\xe0':  # Arrow key prefix
                key = msvcrt.getch()
                if key == b'H':
                    return 'up'
                elif key == b'P':
                    return 'down'
            elif key == b'\r':
                return 'enter'
            elif key in (b'q', b'Q', b'\x03'):
                return 'quit'
            elif key in (b'w', b'W'):
                return 'up'
            elif key in (b's', b'S'):
                return 'down'
            elif key in (b'/', b'?'):
                return 'search'
            elif key in (b'h', b'H'):
                return 'help'
            return key.decode('utf-8', errors='ignore')
        else:
            import tty
            import termios
            fd = sys.stdin.fileno()
            old_settings = termios.tcgetattr(fd)
            try:
                tty.setraw(sys.stdin.fileno())
                ch = sys.stdin.read(1)
                if ch == '\x1b':  # ESC sequence
                    next_chars = sys.stdin.read(2)
                    if next_chars == '[A':
                        return 'up'
                    elif next_chars == '[B':
                        return 'down'
                elif ch in ('\r', '\n'):
                    return 'enter'
                elif ch in ('q', 'Q', '\x03'):
                    return 'quit'
                elif ch in ('w', 'W'):
                    return 'up'
                elif ch in ('s', 'S'):
                    return 'down'
                elif ch in ('/', '?'):
                    return 'search'
                elif ch in ('h', 'H'):
                    return 'help'
                return ch
            finally:
                termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)

    # =================== Dashboard Manager ===================

    class DashboardManager:
        """Manages the interactive dashboard"""

        def __init__(self, app):
            self.app = app
            self.current_view = "main_menu"
            self.selected_index = 0
            self.running = True
            self.history = []
            self.search_query = ""

            # Cache
            self.modules_cache = None
            self.current_module = None
            self.current_functions = []

        async def run(self):
            """Main run loop"""
            # Clear screen
            print('\033[2J\033[H')

            # Welcome
            print_box_header("ToolBoxV2 Interactive Dashboard", "🎯")
            print_box_content("Welcome to the ToolBoxV2 Command Center", "info")
            print_box_footer()

            await asyncio.sleep(1)

            while self.running:
                try:
                    if self.current_view == "main_menu":
                        await self.show_main_menu()
                    elif self.current_view == "modules":
                        await self.show_modules()
                    elif self.current_view == "module_detail":
                        await self.show_module_detail()
                    elif self.current_view == "function_execute":
                        await self.execute_function()
                    elif self.current_view == "function_runner":
                        await self.show_function_runner()
                    elif self.current_view == "workflow_runner":
                        await self.show_workflow_runner()
                    elif self.current_view == "status":
                        await self.show_status()
                    elif self.current_view == "services":
                        await self.show_services()
                    elif self.current_view == "quick_actions":
                        await self.show_quick_actions()
                    elif self.current_view == "search":
                        await self.show_search()
                    elif self.current_view == "settings":
                        await self.show_settings()
                except KeyboardInterrupt:
                    if await self.confirm_exit():
                        break
                    continue

        async def show_main_menu(self):
            """Show main menu"""
            menu_items = [
                ("📦", "Browse Modules", "modules"),
                ("⚡", "Quick Actions", "quick_actions"),
                ("🎯", "Function Runner", "function_runner"),
                ("⏩", "Workflow Runner", "workflow_runner"),
                ("🔧", "Manage Services", "services"),
                ("📊", "System Status", "status"),
                ("🔍", "Search", "search"),
                ("⚙️", "Settings", "settings"),
                ("❌", "Exit", "exit")
            ]

            while True:
                print('\033[2J\033[H')

                print_box_header("Main Menu", "🏠")
                print()

                # User info
                username = self.app.get_username() if hasattr(self.app, 'get_username') else "Guest"
                print(f"  👤 User: {username}")
                print(f"  📍 Instance: {self.app.id}")
                print(f"  🖥️  System: {system()}")
                print()
                print_separator()
                print()

                # Menu items
                for i, (icon, label, _) in enumerate(menu_items):
                    is_selected = i == self.selected_index
                    arrow = "▶" if is_selected else " "

                    if is_selected:
                        print(f"  {arrow} \033[1;96m{icon} {label}\033[0m")
                    else:
                        print(f"  {arrow} {icon} {label}")

                print()
                print_box_footer()
                print_status("↑↓/w/s: Navigate | Enter: Select | h: Help | q: Quit", "info")

                key = get_key()

                if key == 'quit':
                    if await self.confirm_exit():
                        self.running = False
                        return
                elif key == 'up':
                    self.selected_index = max(0, self.selected_index - 1)
                elif key == 'down':
                    self.selected_index = min(len(menu_items) - 1, self.selected_index + 1)
                elif key == 'enter':
                    _, _, action = menu_items[self.selected_index]

                    if action == "exit":
                        if await self.confirm_exit():
                            self.running = False
                            return
                    elif action == "function_runner":
                        self.history.append(self.current_view)
                        self.current_view = "function_runner"
                        self.selected_index = 0
                        return
                    elif action == "workflow_runner":
                        self.history.append(self.current_view)
                        self.current_view = "workflow_runner"
                        self.selected_index = 0
                        return
                    else:
                        self.history.append(self.current_view)
                        self.current_view = action
                        self.selected_index = 0
                        return
                elif key == 'search':
                    self.history.append(self.current_view)
                    self.current_view = "search"
                    return
                elif key == 'help':
                    await self.show_help()

        async def show_modules(self):
            """Show modules list"""
            if self.modules_cache is None:
                print_status("Loading modules...", "progress")
                self.modules_cache = list(self.app.functions.keys())
                self.modules_cache.sort()

            while True:
                print('\033[2J\033[H')

                print_box_header("Module Browser", "📦")
                print_box_content(f"Total modules: {len(self.modules_cache)}", "info")
                print_box_footer()

                if not self.modules_cache:
                    print_status("No modules loaded", "warning")
                    print_status("Use -l flag to load all modules", "info")
                    print()
                    print_status("Press any key to go back...", "info")
                    get_key()
                    self.go_back()
                    return

                # Calculate visible range
                visible_count = 15
                start_idx = max(0, self.selected_index - visible_count // 2)
                end_idx = min(len(self.modules_cache), start_idx + visible_count)

                if end_idx - start_idx < visible_count:
                    start_idx = max(0, end_idx - visible_count)

                # Show modules
                print()
                for i in range(start_idx, end_idx):
                    module_name = self.modules_cache[i]
                    is_selected = i == self.selected_index
                    arrow = "▶" if is_selected else " "

                    # Get module version if available
                    try:
                        mod = self.app.get_mod(module_name)
                        version = getattr(mod, 'version', '?.?.?')
                    except:
                        version = '?.?.?'

                    if is_selected:
                        print(f"  {arrow} \033[1;96m📦 {module_name:<30} v{version}\033[0m")
                    else:
                        print(f"  {arrow} 📦 {module_name:<30} v{version}")

                if len(self.modules_cache) > visible_count:
                    print(f"\n  Showing {start_idx + 1}-{end_idx} of {len(self.modules_cache)}")

                print()
                print_separator()
                print_status("↑↓/w/s: Navigate | Enter: Open | /: Search | b/Esc: Back", "info")

                key = get_key()

                if key in ('quit', 'b', 'B'):
                    self.go_back()
                    return
                elif key == 'up':
                    self.selected_index = max(0, self.selected_index - 1)
                elif key == 'down':
                    self.selected_index = min(len(self.modules_cache) - 1, self.selected_index + 1)
                elif key == 'enter':
                    self.current_module = self.modules_cache[self.selected_index]
                    self.history.append(self.current_view)
                    self.current_view = "module_detail"
                    self.selected_index = 0
                    return
                elif key == 'search':
                    self.history.append(self.current_view)
                    self.current_view = "search"
                    return

        async def show_module_detail(self):
            """Show module detail with functions"""
            if not self.current_module:
                self.go_back()
                return

            # Load functions
            print_status(f"Loading functions from {self.current_module}...", "progress")

            module_data = self.app.functions.get(self.current_module, {})
            self.current_functions = []

            for func_name, func_data in module_data.items():
                if isinstance(func_data, dict) and 'func' in func_data:
                    self.current_functions.append({
                        'name': func_name,
                        'data': func_data
                    })

            while True:
                print('\033[2J\033[H')

                print_box_header(f"Module: {self.current_module}", "📦")

                # Module info
                try:
                    mod = self.app.get_mod(self.current_module)
                    version = getattr(mod, 'version', 'unknown')
                    print_box_content(f"Version: {version}", "info")
                except:
                    print_box_content("Version: unknown", "warning")

                print_box_content(f"Functions: {len(self.current_functions)}", "info")
                print_box_footer()
                print()

                if not self.current_functions:
                    print_status("No functions available in this module", "warning")
                    print()
                    print_status("Press any key to go back...", "info")
                    get_key()
                    self.go_back()
                    return

                # Show functions
                visible_count = 12
                start_idx = max(0, self.selected_index - visible_count // 2)
                end_idx = min(len(self.current_functions), start_idx + visible_count)

                if end_idx - start_idx < visible_count:
                    start_idx = max(0, end_idx - visible_count)

                for i in range(start_idx, end_idx):
                    func = self.current_functions[i]
                    is_selected = i == self.selected_index
                    arrow = "▶" if is_selected else " "

                    # Get function type
                    func_type = func['data'].get('type', 'unknown')
                    type_icon = "⚡" if 'async' in str(func_type) else "🔧"

                    if is_selected:
                        print(f"  {arrow} \033[1;96m{type_icon} {func['name']}\033[0m")
                    else:
                        print(f"  {arrow} {type_icon} {func['name']}")

                if len(self.current_functions) > visible_count:
                    print(f"\n  Showing {start_idx + 1}-{end_idx} of {len(self.current_functions)}")

                print()
                print_separator()
                print_status("↑↓/w/s: Navigate | Enter: Execute | i: Info | b: Back", "info")

                key = get_key()

                if key in ('quit', 'b', 'B'):
                    self.go_back()
                    return
                elif key == 'up':
                    self.selected_index = max(0, self.selected_index - 1)
                elif key == 'down':
                    self.selected_index = min(len(self.current_functions) - 1, self.selected_index + 1)
                elif key == 'enter':
                    self.history.append(self.current_view)
                    self.current_view = "function_execute"
                    return
                elif key in ('i', 'I'):
                    await self.show_function_info(self.current_functions[self.selected_index])

        async def show_function_runner(self):
            """Interactive function runner with autocomplete"""
            print('\033[2J\033[H')

            print_box_header("Function Runner", "🎯")
            print_box_content("Execute functions with autocomplete", "info")
            print_box_footer()

            # Restore terminal for input
            if system() != "Windows":
                import termios
                fd = sys.stdin.fileno()
                old_settings = termios.tcgetattr(fd)
                termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)

            print()
            print("  Format: module_name function_name [args...]")
            print("  Example: CloudM Version")
            print("  Example: helper create-user john john@mail.com")
            print()

            # Get all available modules and functions for autocomplete hints
            available_modules = list(self.app.functions.keys())

            print("  Available modules:")
            print("  " + ", ".join(available_modules[:10]))
            if len(available_modules) > 10:
                print(f"  ... and {len(available_modules) - 10} more")
            print()

            command_input = input("  Command: ").strip()

            if not command_input:
                self.go_back()
                return

            parts = command_input.split()

            if len(parts) < 2:
                print()
                print_status("Need at least module and function name", "error")
                print_status("Press any key to continue...", "info")
                get_key()
                return

            module_name = parts[0]
            function_name = parts[1]
            args = parts[2:]

            # Check if module exists
            if module_name not in self.app.functions:
                print()
                print_status(f"Module '{module_name}' not found", "error")

                # Suggest similar modules
                similar = [m for m in available_modules if module_name.lower() in m.lower()]
                if similar:
                    print()
                    print("  Did you mean:")
                    for s in similar[:5]:
                        print(f"    • {s}")

                print()
                print_status("Press any key to continue...", "info")
                get_key()
                return

            # Check if function exists
            module_data = self.app.functions.get(module_name, {})
            if function_name not in module_data:
                print()
                print_status(f"Function '{function_name}' not found in {module_name}", "error")

                # Show available functions
                available_funcs = [f for f in module_data.keys() if isinstance(module_data[f], dict)]
                if available_funcs:
                    print()
                    print("  Available functions:")
                    for f in available_funcs[:10]:
                        print(f"    • {f}")
                    if len(available_funcs) > 10:
                        print(f"    ... and {len(available_funcs) - 10} more")

                print()
                print_status("Press any key to continue...", "info")
                get_key()
                return

            # Ask for kwargs
            print()
            print_status("Enter keyword arguments (optional)", "info")
            kwargs_input = input("  Kwargs (key=value, space-separated): ").strip()

            kwargs = {}
            if kwargs_input:
                for pair in kwargs_input.split():
                    if '=' in pair:
                        key, value = pair.split('=', 1)
                        kwargs[key.strip()] = value.strip()
                    elif ':' in pair:
                        key, value = pair.split(':', 1)
                        kwargs[key.strip()] = value.strip()

            print()
            print_separator("═")
            print(f"  Executing: {module_name}.{function_name}")
            print_separator("═")
            print()

            try:
                # Execute function
                result = await self.app.a_run_any(
                    (module_name, function_name),
                    args_=args,
                    tb_run_with_specification='app',
                    get_results=True,
                    **kwargs
                )

                # Handle coroutine results
                if asyncio.iscoroutine(result):
                    result = await result

                if isinstance(result, asyncio.Task):
                    result = await result

                print()
                print_separator("═")
                print("  Result:")
                print_separator("═")
                print()

                if hasattr(result, 'print'):
                    result.print(full_data=True)
                elif hasattr(result, '__dict__'):
                    import pprint
                    pprint.pprint(result.__dict__)
                else:
                    print(f"  {result}")

                print()
                print_status("Execution completed successfully", "success")

            except Exception as e:
                print()
                print_status(f"Execution failed: {e}", "error")

                import traceback
                print()
                print("  Traceback:")
                print_separator()
                traceback.print_exc()

            print()
            print_status("Press any key to continue...", "info")
            get_key()

            # Ask if user wants to run another command
            if system() != "Windows":
                import termios
                fd = sys.stdin.fileno()
                old_settings = termios.tcgetattr(fd)
                termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)

            again = input("\n  Run another command? (y/N): ").strip().lower()

            if again == 'y':
                # Stay in function runner
                return
            else:
                self.go_back()

        async def show_workflow_runner(self):
            """Interactive workflow runner with autocomplete"""
            print('\033[2J\033[H')

            print_box_header("Workflow Runner", "🎯")
            print_box_content("Execute workflows with autocomplete", "info")
            print_box_footer()

            if system() != "Windows":
                import termios
                fd = sys.stdin.fileno()
                old_settings = termios.tcgetattr(fd)
                termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)

            all_flows = self.app.flows.keys()
            if not all_flows:
                from toolboxv2.flows import flows_dict as flows_dict_func
                flows_dict = flows_dict_func(remote=False)
                self.app.set_flows(flows_dict)
                all_flows = self.app.flows.keys()
            print("  Available workflows:")
            # show in an 3 by n grid
            for i, flow in enumerate(all_flows):
                print(f" {str(i) + ' '+flow:<20}", end='\n' if i % 3 == 2 else ' ')

            command_input = input("  Workflow: ").strip()

            try:
                command_input = int(command_input)
                command_input = list(all_flows)[command_input]
            except:
                pass

            if not command_input:
                self.go_back()
                return

            if command_input not in all_flows:
                print()
                print_status(f"Workflow '{command_input}' not found", "error")
                print_status("Press any key to continue...", "info")
                get_key()
                return

            print()
            print_separator("═")
            print(f"  Executing: {command_input}")
            print_separator("═")
            print()
            try:
                self.go_back()
                await self.app.run_flows(command_input)
                print()
                print_status("Execution completed successfully", "success")
            except Exception as e:
                print()
                print_status(f"Execution failed: {e}", "error")
                import traceback
                print()
                print("  Traceback:")
                print_separator()
                traceback.print_exc()


        async def execute_function(self):
            """Execute selected function"""
            if not self.current_module or not self.current_functions:
                self.go_back()
                return

            func = self.current_functions[self.selected_index]

            print('\033[2J\033[H')

            print_box_header(f"Execute Function", "⚡")
            print_box_content(f"Module: {self.current_module}", "info")
            print_box_content(f"Function: {func['name']}", "info")
            print_box_footer()

            # Restore terminal for input
            if system() != "Windows":
                import termios
                fd = sys.stdin.fileno()
                old_settings = termios.tcgetattr(fd)
                termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)

            # Get arguments
            print()
            print_status("Enter function arguments (leave empty if none)", "info")
            args_input = input("  Args (space-separated): ").strip()

            args = args_input.split() if args_input else []

            print()
            print_status("Enter keyword arguments (leave empty if none)", "info")
            kwargs_input = input("  Kwargs (key=value, space-separated): ").strip()

            kwargs = {}
            if kwargs_input:
                for pair in kwargs_input.split():
                    if '=' in pair:
                        key, value = pair.split('=', 1)
                        kwargs[key.strip()] = value.strip()

            print()
            print_separator("═")
            print("  Executing...")
            print_separator("═")
            print()

            try:
                # Execute
                result = await self.app.a_run_any(
                    (self.current_module, func['name']),
                    args_=args,
                    tb_run_with_specification='app',
                    get_results=True,
                    **kwargs
                )

                if asyncio.iscoroutine(result):
                    result = await result

                print()
                print_separator("═")
                print("  Result:")
                print_separator("═")
                print()

                if hasattr(result, 'print'):
                    result.print(full_data=True)
                else:
                    print(f"  {result}")

                print()
                print_status("Execution completed successfully", "success")

            except Exception as e:
                print()
                print_status(f"Execution failed: {e}", "error")

                import traceback
                print()
                print("  Traceback:")
                print_separator()
                print(traceback.format_exc())

            print()
            print_status("Press any key to continue...", "info")
            get_key()

            self.go_back()

        async def show_status(self):
            """Show system status"""
            print('\033[2J\033[H')

            print_box_header("System Status", "📊")

            # User info
            try:
                username = self.app.get_username() if hasattr(self.app, 'get_username') else "Guest"
                login_status = "Not logged in"
                login_style = "error"

                # Check login status
                try:
                    from toolboxv2.utils.extras.blobs import BlobFile
                    from toolboxv2.utils.security.cryp import Code

                    with BlobFile(f"claim/{username}/jwt.c", key=Code.DK()(), mode="r") as blob:
                        claim = blob.read()
                        if claim and claim != b'Error decoding':
                            login_status = "Logged in"
                            login_style = "success"
                except:
                    pass
            except:
                username = "Guest"
                login_status = "Not logged in"
                login_style = "error"

            print_box_content(f"User: {username}", "info")
            print_box_content(f"Status: {login_status}", login_style)
            print_box_content(f"Instance: {self.app.id}", "info")
            print_box_content(f"System: {system()} on {node()}", "info")

            # Modules
            modules_count = len(self.app.functions.keys())
            print_box_content(f"Loaded Modules: {modules_count}", "info")

            # Services Status
            print_separator("─")

            # Check DB
            db_status = "Available"
            db_style = "success"
            try:
                # Quick check without full status output
                pass
            except:
                db_status = "Not available"
                db_style = "error"

            print_box_content(f"Database: {db_status}", db_style)

            # Check API
            api_status = "Stopped"
            api_style = "error"
            api_info = ""
            try:
                from toolboxv2.utils.system.state_system import read_server_state
                pid, _, _ = read_server_state()
                from toolboxv2.utils.system.state_system import is_process_running
                if is_process_running(pid):
                    api_status = "Running"
                    api_style = "success"
                    api_info = f" (PID: {pid})"
            except:
                pass

            print_box_content(f"API Server: {api_status}{api_info}", api_style)

            print_box_footer()

            print_status("Press any key to go back...", "info")
            get_key()

            self.go_back()

        async def show_services(self):
            """Show services management"""
            services = [
                ("🖥️", "API Server", "api"),
                ("🗄️", "Database", "db"),
                ("🌐", "P2P Client", "p2p"),
                ("📦", "Module Manager", "mods"),
                ("🔙", "Back", "back")
            ]

            while True:
                print('\033[2J\033[H')

                print_box_header("Service Management", "🔧")
                print_box_footer()

                print()
                for i, (icon, label, _) in enumerate(services):
                    is_selected = i == self.selected_index
                    arrow = "▶" if is_selected else " "

                    if is_selected:
                        print(f"  {arrow} \033[1;96m{icon} {label}\033[0m")
                    else:
                        print(f"  {arrow} {icon} {label}")

                print()
                print_separator()
                print_status("↑↓/w/s: Navigate | Enter: Manage | b: Back", "info")

                key = get_key()

                if key in ('quit', 'b', 'B'):
                    self.go_back()
                    return
                elif key == 'up':
                    self.selected_index = max(0, self.selected_index - 1)
                elif key == 'down':
                    self.selected_index = min(len(services) - 1, self.selected_index + 1)
                elif key == 'enter':
                    _, _, action = services[self.selected_index]

                    if action == "back":
                        self.go_back()
                        return
                    else:
                        await self.manage_service(action)

        async def manage_service(self, service_name: str):
            """Manage a specific service"""
            print('\033[2J\033[H')

            print_box_header(f"Manage {service_name.upper()}", "🔧")
            print_box_footer()

            actions = [
                ("▶️", "Start", "start"),
                ("⏹️", "Stop", "stop"),
                ("📊", "Status", "status"),
            ]

            if service_name == "api":
                # build, debug, clean, remove-exe, update
                actions.extend([
                    ("🔨", "Build", "build"),
                    ("🔥", "Debug", "debug"),
                    ("🧹", "Clean", "clean"),
                    ("🗑️", "Remove Executable", "remove-exe"),
                    ("🔄", "Update", "update"),
                ])
            if service_name == "db":
                # health, update , build, clean, discover
                actions.extend([
                    ("❤️", "Health", "health"),
                    ("🔄", "Update", "update"),
                    ("🔨", "Build", "build"),
                    ("🧹", "Clean", "clean"),
                    ("🔍", "Discover", "discover"),
                ])
            if service_name == "p2p":
                # interactive
                actions.append(("🎮", "Interactive", "interactive"))
                actions.remove(("▶️", "Start", "start"))
                actions.remove(("⏹️", "Stop", "stop"))

            actions.append(("🔙", "Back", "back"))

            action_idx = 0

            while True:
                print('\033[2J\033[H')

                print_box_header(f"Manage {service_name.upper()}", "🔧")
                print_box_footer()

                print()
                for i, (icon, label, _) in enumerate(actions):
                    is_selected = i == action_idx
                    arrow = "▶" if is_selected else " "

                    if is_selected:
                        print(f"  {arrow} \033[1;96m{icon} {label}\033[0m")
                    else:
                        print(f"  {arrow} {icon} {label}")

                print()
                print_separator()
                print_status("↑↓/w/s: Navigate | Enter: Execute | b: Back", "info")

                key = get_key()

                if key in ('quit', 'b', 'B'):
                    return
                elif key == 'up':
                    action_idx = max(0, action_idx - 1)
                elif key == 'down':
                    action_idx = min(len(actions) - 1, action_idx + 1)
                elif key == 'enter':
                    _, _, action = actions[action_idx]

                    if action == "back":
                        return

                    print()
                    print_separator("═")
                    print(f"  Executing: {action} on {service_name}")
                    print_separator("═")
                    print()

                    # Execute action
                    try:
                        if service_name == "api":
                            sys.argv = ["api", action]
                            cli_api_runner()
                        elif service_name == "db":
                            sys.argv = ["db", action]
                            cli_db_runner()
                        elif service_name == "p2p":
                            sys.argv = ["p2p", action]
                            cli_tcm_runner()
                        elif service_name == "mods":
                            await self.app.a_run_any("CloudM", "manager")

                        print()
                        print_status("Command executed", "success")
                    except Exception as e:
                        print()
                        print_status(f"Error: {e}", "error")

                    print()
                    print_status("Press any key to continue...", "info")
                    get_key()

        async def show_quick_actions(self):
            """Show quick actions menu"""
            actions = [
                ("🔐", "Login", self.quick_login),
                ("🚪", "Logout", self.quick_logout),
                ("📊", "System Status", self.quick_status),
                ("🔄", "Reload Modules", self.quick_reload),
                ("🧹", "Clear Cache", self.quick_clear_cache),
                ("🔙", "Back", None)
            ]

            while True:
                print('\033[2J\033[H')

                print_box_header("Quick Actions", "⚡")
                print_box_footer()

                print()
                for i, (icon, label, _) in enumerate(actions):
                    is_selected = i == self.selected_index
                    arrow = "▶" if is_selected else " "

                    if is_selected:
                        print(f"  {arrow} \033[1;96m{icon} {label}\033[0m")
                    else:
                        print(f"  {arrow} {icon} {label}")

                print()
                print_separator()
                print_status("↑↓/w/s: Navigate | Enter: Execute | b: Back", "info")

                key = get_key()

                if key in ('quit', 'b', 'B'):
                    self.go_back()
                    return
                elif key == 'up':
                    self.selected_index = max(0, self.selected_index - 1)
                elif key == 'down':
                    self.selected_index = min(len(actions) - 1, self.selected_index + 1)
                elif key == 'enter':
                    _, _, action_func = actions[self.selected_index]

                    if action_func is None:
                        self.go_back()
                        return

                    await action_func()

        async def show_search(self):
            """Show search interface"""
            print('\033[2J\033[H')

            print_box_header("Search", "🔍")
            print_box_footer()

            # Restore terminal for input
            if system() != "Windows":
                import termios
                fd = sys.stdin.fileno()
                old_settings = termios.tcgetattr(fd)
                termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)

            print()
            query = input("  Search query: ").strip().lower()

            if not query:
                self.go_back()
                return

            print()
            print_status("Searching...", "progress")

            # Search in modules and functions
            results = []

            for module_name in self.app.functions.keys():
                if query in module_name.lower():
                    results.append(("module", module_name, None))

                module_data = self.app.functions.get(module_name, {})
                for func_name in module_data.keys():
                    if query in func_name.lower():
                        results.append(("function", module_name, func_name))

            print('\033[2J\033[H')

            print_box_header(f"Search Results for '{query}'", "🔍")
            print_box_content(f"Found {len(results)} results", "info")
            print_box_footer()

            if not results:
                print()
                print_status("No results found", "warning")
            else:
                print()
                for result_type, module, func in results[:20]:  # Limit to 20 results
                    if result_type == "module":
                        print(f"  📦 Module: {module}")
                    else:
                        print(f"  ⚡ Function: {module}.{func}")

                if len(results) > 20:
                    print(f"\n  ... and {len(results) - 20} more results")

            print()
            print_separator()
            print_status("Press any key to go back...", "info")
            get_key()

            self.go_back()

        async def show_settings(self):
            """Show settings menu"""
            settings = [
                ("🔧", "Environment Variables", "env"),
                ("📝", "View Config", "view_config"),
                ("💾", "Save Config", "save_config"),
                ("📈", "App Footprint", "app_footprint"),
                ("ℹ️", "About", "about"),

                ("🔙", "Back", "back")
            ]

            self.selected_index = 0

            while True:
                print('\033[2J\033[H')

                print_box_header("Settings", "⚙️")
                print_box_footer()

                print()
                for i, (icon, label, _) in enumerate(settings):
                    is_selected = i == self.selected_index
                    arrow = "▶" if is_selected else " "

                    if is_selected:
                        print(f"  {arrow} \033[1;96m{icon} {label}\033[0m")
                    else:
                        print(f"  {arrow} {icon} {label}")

                print()
                print_separator()
                print_status("↑↓/w/s: Navigate | Enter: Open | b: Back", "info")

                key = get_key()

                if key in ('quit', 'b', 'B'):
                    self.go_back()
                    return
                elif key == 'up':
                    self.selected_index = max(0, self.selected_index - 1)
                elif key == 'down':
                    self.selected_index = min(len(settings) - 1, self.selected_index + 1)
                elif key == 'enter':
                    _, _, action = settings[self.selected_index]

                    if action == "back":
                        self.go_back()
                        return
                    elif action == "about":
                        await self.show_about()
                    elif action == "env":
                        await self.manage_env_vars()
                    elif action == "view_config":
                        await self.view_config()
                    elif action == "save_config":
                        await self.save_config()
                    elif action == "app_footprint":
                        print(get_app().print_footprint())
                        input(Style.GREY("Press Enter to continue..."))

        async def manage_env_vars(self):
            """Manage environment variables"""
            import os

            # Important ToolBox env vars
            env_vars = [
                ("TOOLBOXV2_REMOTE_BASE", "Remote server base URL", os.getenv("TOOLBOXV2_REMOTE_BASE", "")),
                ("APP_BASE_URL", "Application base URL", os.getenv("APP_BASE_URL", "")),
                ("TB_R_KEY", "Remote access key", os.getenv("TB_R_KEY", "")),
                ("DB_MODE_KEY", "Database mode", os.getenv("DB_MODE_KEY", "LC")),
                ("PYTHON_EXECUTABLE", "Python executable path", os.getenv("PYTHON_EXECUTABLE", "")),
                ("RUST_LOG", "Rust log level", os.getenv("RUST_LOG", "")),
            ]

            actions = [
                ("➕", "Add/Edit Variable", "edit"),
                ("📋", "View All", "view"),
                ("💾", "Save to .env", "save"),
                ("🔄", "Reload from .env", "reload"),
                ("🔙", "Back", "back")
            ]

            selected = 0

            while True:
                print('\033[2J\033[H')

                print_box_header("Environment Variables", "🔧")

                # Build ENV format string for display
                env_content = ""
                for var_name, description, value in env_vars:
                    env_content += f"# {description}\n"
                    if value:
                        env_content += f"{var_name}={value}\n"
                    else:
                        env_content += f"# {var_name}=(not set)\n"
                    env_content += "\n"

                # Display as formatted ENV file
                print_code_block(env_content.strip(), "env", show_line_numbers=False)
                print_box_footer()

                print()
                print_separator("─")
                print("  Actions:")
                print_separator("─")
                print()

                for i, (icon, label, _) in enumerate(actions):
                    is_selected = i == selected
                    arrow = "▶" if is_selected else " "

                    if is_selected:
                        print(f"  {arrow} \033[1;96m{icon} {label}\033[0m")
                    else:
                        print(f"  {arrow} {icon} {label}")

                print()
                print_separator()
                print_status("↑↓/w/s: Navigate | Enter: Select | b: Back", "info")

                key = get_key()

                if key in ('quit', 'b', 'B'):
                    return
                elif key == 'up':
                    selected = max(0, selected - 1)
                elif key == 'down':
                    selected = min(len(actions) - 1, selected + 1)
                elif key == 'enter':
                    _, _, action = actions[selected]

                    if action == "back":
                        return
                    elif action == "edit":
                        await self.edit_env_var(env_vars)
                    elif action == "view":
                        await self.view_all_env_vars()
                    elif action == "save":
                        await self.save_env_to_file(env_vars)
                    elif action == "reload":
                        await self.reload_env_from_file()

        async def edit_env_var(self, env_vars):
            """Edit an environment variable"""
            print('\033[2J\033[H')

            print_box_header("Edit Environment Variable", "✎")
            print_box_footer()

            # Restore terminal
            if system() != "Windows":
                import termios
                fd = sys.stdin.fileno()
                old_settings = termios.tcgetattr(fd)
                termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)

            print()
            print("  Available variables:")
            for i, (name, desc, _) in enumerate(env_vars, 1):
                print(f"    {i}. {name} - {desc}")

            print()
            choice = input("  Select variable number (or enter custom name): ").strip()

            try:
                idx = int(choice) - 1
                if 0 <= idx < len(env_vars):
                    var_name = env_vars[idx][0]
                else:
                    var_name = choice
            except ValueError:
                var_name = choice

            if not var_name:
                return

            current_value = os.getenv(var_name, "")
            print(f"\n  Current value: {current_value or '(not set)'}")

            new_value = input(f"  New value (leave empty to keep current): ").strip()

            if new_value:
                os.environ[var_name] = new_value
                print()
                print_status(f"Set {var_name} = {new_value}", "success")
            else:
                print()
                print_status("No changes made", "info")

            print()
            print_status("Press any key to continue...", "info")
            get_key()

        async def view_all_env_vars(self):
            """View all environment variables"""
            print('\033[2J\033[H')

            print_box_header("All Environment Variables", "📋")
            print_box_footer()

            env_vars = sorted(os.environ.items())

            print()
            print(f"  Total: {len(env_vars)} variables")
            print()
            print_separator()

            # Show first 30
            for key, value in env_vars[:30]:
                display_value = value
                if len(display_value) > 50:
                    display_value = display_value[:47] + "..."
                print(f"  {key:<30} = {display_value}")

            if len(env_vars) > 30:
                print(f"\n  ... and {len(env_vars) - 30} more")

            print()
            print_separator()
            print_status("Press any key to go back...", "info")
            get_key()

        async def save_env_to_file(self, env_vars):
            """Save environment variables to .env file"""
            from pathlib import Path

            print('\033[2J\033[H')

            print_box_header("Save to .env File", "💾")
            print_box_footer()

            env_file = Path(".env")

            print()
            print(f"  File: {env_file.absolute()}")
            print()

            # Restore terminal
            if system() != "Windows":
                import termios
                fd = sys.stdin.fileno()
                old_settings = termios.tcgetattr(fd)
                termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)

            confirm = input("  Save current values to .env? (y/N): ").strip().lower()

            if confirm == 'y':
                try:
                    with open(env_file, 'w') as f:
                        for var_name, description, value in env_vars:
                            current = os.getenv(var_name, value)
                            if current:
                                f.write(f"# {description}\n")
                                f.write(f"{var_name}={current}\n\n")

                    print()
                    print_status(f"Saved to {env_file}", "success")
                except Exception as e:
                    print()
                    print_status(f"Error saving: {e}", "error")
            else:
                print()
                print_status("Cancelled", "info")

            print()
            print_status("Press any key to continue...", "info")
            get_key()

        async def reload_env_from_file(self):
            """Reload environment variables from .env file"""
            from pathlib import Path
            from dotenv import load_dotenv

            print('\033[2J\033[H')

            print_box_header("Reload from .env", "🔄")
            print_box_footer()

            env_file = Path(".env")

            print()
            if not env_file.exists():
                print_status(f".env file not found: {env_file.absolute()}", "warning")
            else:
                try:
                    load_dotenv(override=True)
                    print_status("Environment variables reloaded", "success")
                except Exception as e:
                    print_status(f"Error reloading: {e}", "error")

            print()
            print_status("Press any key to continue...", "info")
            get_key()

        async def view_config(self):
            """View current configuration"""
            import json
            print('\033[2J\033[H')

            print_box_header("Current Configuration", "📝")

            # Build configuration as JSON
            modules_count = len(self.app.functions.keys())
            config_data = {
                "application": {
                    "instance_id": self.app.id,
                    "start_directory": str(self.app.start_dir),
                    "system": system(),
                    "node": node()
                },
                "modules": {
                    "loaded_count": modules_count,
                    "names": sorted(list(self.app.functions.keys())[:10])  # Show first 10
                },
                "environment": {
                    "remote_base": os.getenv("TOOLBOXV2_REMOTE_BASE", "not set"),
                    "app_base_url": os.getenv("APP_BASE_URL", "not set"),
                    "db_mode": os.getenv("DB_MODE_KEY", "LC")
                }
            }

            # Display as formatted JSON
            config_json = json.dumps(config_data, indent=2)
            print_code_block(config_json, "json", show_line_numbers=True)

            if modules_count > 10:
                print_box_content(f"... and {modules_count - 10} more modules", "info")

            print_box_footer()

            print_status("Press any key to go back...", "info")
            get_key()

        async def save_config(self):
            """Save current configuration"""
            print('\033[2J\033[H')

            print_box_header("Save Configuration", "💾")
            print_box_footer()

            print()
            print_status("Configuration auto-saved", "success")
            print()
            print_status("Press any key to continue...", "info")
            get_key()

        async def show_about(self):
            """Show about information"""
            print('\033[2J\033[H')

            print_box_header("About ToolBoxV2", "ℹ️")

            version = get_version_from_pyproject()
            from toolboxv2 import tb_root_dir, init_cwd

            print_box_content("ToolBoxV2 Interactive Dashboard", "info")
            print_box_content(f"Version: {version}", "info")
            print_box_content(f"System: {system()}", "info")
            print_box_content(f"Python: {sys.version.split()[0]}", "info")
            print_separator("─")
            print_box_content(f"Home: {tb_root_dir}", "info")
            print_box_content(f"Start: {init_cwd}", "info")
            print(f"  A powerful, modular Python framework")
            print(f"  for building and managing tools.")
            print()
            print_box_footer()

            print_status("Press any key to go back...", "info")
            get_key()

        async def show_help(self):
            """Show help screen"""
            print('\033[2J\033[H')

            print_box_header("Keyboard Shortcuts", "❓")
            print()
            print("  Navigation:")
            print("    ↑/↓ or w/s     Navigate menu items")
            print("    Enter          Select/Execute")
            print("    b / Esc        Go back")
            print()
            print("  Global:")
            print("    /              Search")
            print("    h              Show help")
            print("    q              Quit")
            print()
            print("  Function Execution:")
            print("    i              Show function info")
            print("    Enter          Execute function")
            print()
            print_box_footer()

            print_status("Press any key to continue...", "info")
            get_key()

        async def show_function_info(self, func):
            """Show detailed function information"""
            import json
            print('\033[2J\033[H')

            print_box_header(f"Function Info: {func['name']}", "ℹ️")

            func_data = func['data']

            # Build function info as JSON
            info_data = {
                "name": func['name'],
                "module": self.current_module,
                "type": func_data.get('type', 'unknown'),
            }

            if 'version' in func_data:
                info_data['version'] = func_data['version']

            if 'test' in func_data:
                info_data['testable'] = func_data['test']

            # Display as formatted JSON
            info_json = json.dumps(info_data, indent=2)
            print_code_block(info_json, "json", show_line_numbers=False)

            # Try to get docstring
            try:
                func_obj = func_data.get('func')
                if func_obj and hasattr(func_obj, '__doc__') and func_obj.__doc__:
                    print_separator("─")
                    print_box_content("Description:", "info")
                    docstring = func_obj.__doc__.strip()
                    # Display docstring with auto-wrap
                    for line in docstring.split('\n'):
                        if line.strip():
                            print_box_content(line.strip(), auto_wrap=True)
            except:
                pass

            print_box_footer()

            print_status("Press any key to continue...", "info")
            get_key()

        # Quick action implementations

        async def quick_login(self):
            """Quick login action"""
            print('\033[2J\033[H')
            print_box_header("Quick Login", "🔐")
            print_box_footer()

            try:
                result = await self.app.a_run_any("CloudM", "cli_web_login")
                print()
                if result:
                    print_status("Login successful!", "success")
                else:
                    print_status("Login failed or cancelled", "warning")
            except Exception as e:
                print_status(f"Error: {e}", "error")

            print()
            print_status("Press any key to continue...", "info")
            get_key()

        async def quick_logout(self):
            """Quick logout action"""
            print('\033[2J\033[H')
            print_box_header("Quick Logout", "🚪")
            print_box_footer()

            try:
                result = await self.app.a_run_any("CloudM", "cli_logout")
                print()
                print_status("Logout successful!", "success")
            except Exception as e:
                print_status(f"Error: {e}", "error")

            print()
            print_status("Press any key to continue...", "info")
            get_key()

        async def quick_status(self):
            """Quick status check"""
            await self.show_status()

        async def quick_reload(self):
            """Quick module reload"""
            print('\033[2J\033[H')
            print_box_header("Reload Modules", "🔄")
            print_box_footer()

            print()
            print_status("Reloading modules...", "progress")

            try:
                await self.app.load_all_mods_in_file()
                self.modules_cache = None  # Clear cache
                print_status("Modules reloaded successfully!", "success")
            except Exception as e:
                print_status(f"Error: {e}", "error")

            print()
            print_status("Press any key to continue...", "info")
            get_key()

        async def quick_clear_cache(self):
            """Clear dashboard cache"""
            print('\033[2J\033[H')
            print_box_header("Clear Cache", "🧹")
            print_box_footer()

            print()
            self.modules_cache = None
            self.current_functions = []
            print_status("Cache cleared!", "success")

            print()
            print_status("Press any key to continue...", "info")
            get_key()

        # Helper methods

        def go_back(self):
            """Go back to previous view"""
            if self.history:
                self.current_view = self.history.pop()
                self.selected_index = 0
            else:
                self.current_view = "main_menu"
                self.selected_index = 0

        async def confirm_exit(self):
            """Confirm exit"""
            print('\033[2J\033[H')

            print_box_header("Confirm Exit", "❓")
            print_box_content("Are you sure you want to exit?", "warning")
            print_box_footer()

            print()
            print("  Press 'y' to confirm, any other key to cancel")

            key = get_key()
            return key in ('y', 'Y')

    # =================== Main Entry Point ===================

    async def run_dashboard():
        """Run the dashboard"""
        # Setup app
        app= get_app(from_="run_dashboard")

        # Create and run dashboard
        dashboard = DashboardManager(app)

        # Load modules if not already loaded
        if not app.functions or len(app.functions) == 0:
            print_status("No modules loaded. Use -l flag to load all modules.", "info")
            print_status("or in ui '⚡ Quick Actions' -> '🔄 Reload Modules' ", "info")

        await dashboard.run()

        # Cleanup
        print('\033[2J\033[H')
        print_box_header("Goodbye!", "👋")
        print_box_content("Thank you for using ToolBoxV2", "success")
        print_box_footer()

        if not app.called_exit[0]:
            await app.a_exit()

    # Run
    await run_dashboard()

venv_runner

Modern Package Manager Runner - Supporting conda, uv, and native Python

BasePackageManager

Base class for package managers.

Source code in toolboxv2/utils/clis/venv_runner.py

class BasePackageManager:
    """Base class for package managers."""

    def __init__(self, runner: CommandRunner):
        self.runner = runner

    def create_env(self, env_name: str, python_version: str = "3.11") -> bool:
        raise NotImplementedError

    def delete_env(self, env_name: str) -> bool:
        raise NotImplementedError

    def list_envs(self) -> List[str]:
        raise NotImplementedError

    def install_package(self, env_name: str, package: str) -> bool:
        raise NotImplementedError

    def update_package(self, env_name: str, package: str) -> bool:
        raise NotImplementedError

    def list_packages(self, env_name: str) -> List[Dict[str, str]]:
        raise NotImplementedError

    def run_script(self, env_name: str, script: str, args: List[str], python: bool = True) -> bool:
        raise NotImplementedError

CommandRunner

Enhanced command runner with better output handling.

Source code in toolboxv2/utils/clis/venv_runner.py

class CommandRunner:
    """Enhanced command runner with better output handling."""

    def __init__(self, package_manager: PackageManager):
        self.pm = package_manager
        self.encoding = get_encoding()

    def run(self, command: str, live: bool = True, capture: bool = False) -> Tuple[bool, Optional[str]]:
        """
        Execute command with optional live output.

        Args:
            command: Command to execute
            live: Stream output in real-time
            capture: Capture and return output

        Returns:
            Tuple of (success, output)
        """
        print_status('running', f'Executing: {command}')

        if live and not capture:
            # Stream output live
            try:
                process = subprocess.Popen(
                    command,
                    shell=True,
                    stdout=sys.stdout,
                    stderr=sys.stderr,
                    text=True,
                    encoding=self.encoding,
                    errors='replace'
                )
                process.communicate()
                success = process.returncode == 0

                if success:
                    print_status('success', 'Command completed successfully')
                else:
                    print_status('error', f'Command failed with code {process.returncode}')

                return success, None
            except Exception as e:
                print_status('error', f'Execution error: {e}')
                return False, None

        else:
            # Capture output
            try:
                result = subprocess.run(
                    command,
                    shell=True,
                    check=True,
                    text=True,
                    capture_output=True,
                    encoding=self.encoding,
                    errors='replace'
                )
                print_status('success', 'Command completed')
                return True, result.stdout

            except subprocess.CalledProcessError as e:
                print_status('error', 'Command failed')
                if e.stdout:
                    print(f"\nOutput:\n{e.stdout}")
                if e.stderr:
                    print(f"\nError:\n{e.stderr}")
                return False, None

            except Exception as e:
                print_status('error', f'Execution error: {e}')
                return False, None

run(command, live=True, capture=False)

Execute command with optional live output.

Parameters:

Name	Type	Description	Default
command	str	Command to execute	required
live	bool	Stream output in real-time	True
capture	bool	Capture and return output	False

Returns:

Type	Description
Tuple[bool, Optional[str]]	Tuple of (success, output)

Source code in toolboxv2/utils/clis/venv_runner.py

def run(self, command: str, live: bool = True, capture: bool = False) -> Tuple[bool, Optional[str]]:
    """
    Execute command with optional live output.

    Args:
        command: Command to execute
        live: Stream output in real-time
        capture: Capture and return output

    Returns:
        Tuple of (success, output)
    """
    print_status('running', f'Executing: {command}')

    if live and not capture:
        # Stream output live
        try:
            process = subprocess.Popen(
                command,
                shell=True,
                stdout=sys.stdout,
                stderr=sys.stderr,
                text=True,
                encoding=self.encoding,
                errors='replace'
            )
            process.communicate()
            success = process.returncode == 0

            if success:
                print_status('success', 'Command completed successfully')
            else:
                print_status('error', f'Command failed with code {process.returncode}')

            return success, None
        except Exception as e:
            print_status('error', f'Execution error: {e}')
            return False, None

    else:
        # Capture output
        try:
            result = subprocess.run(
                command,
                shell=True,
                check=True,
                text=True,
                capture_output=True,
                encoding=self.encoding,
                errors='replace'
            )
            print_status('success', 'Command completed')
            return True, result.stdout

        except subprocess.CalledProcessError as e:
            print_status('error', 'Command failed')
            if e.stdout:
                print(f"\nOutput:\n{e.stdout}")
            if e.stderr:
                print(f"\nError:\n{e.stderr}")
            return False, None

        except Exception as e:
            print_status('error', f'Execution error: {e}')
            return False, None

CondaManager

Conda package manager implementation.

Source code in toolboxv2/utils/clis/venv_runner.py

class CondaManager(BasePackageManager):
    """Conda package manager implementation."""

    def create_env(self, env_name: str, python_version: str = "3.11") -> bool:
        command = f"conda create -n {env_name} python={python_version} -y"
        return self.runner.run(command)[0]

    def delete_env(self, env_name: str) -> bool:
        command = f"conda env remove -n {env_name} -y"
        success = self.runner.run(command)[0]

        # Clean up registry
        registry_file = Path(f"{env_name}_registry.json")
        if registry_file.exists():
            registry_file.unlink()
            print_status('info', f'Removed registry file: {registry_file}')

        return success

    def list_envs(self) -> List[str]:
        command = "conda env list --json"
        success, output = self.runner.run(command, live=False, capture=True)

        if success and output:
            try:
                data = json.loads(output)
                envs = [Path(env).name for env in data.get('envs', [])]
                return envs
            except json.JSONDecodeError:
                print_status('error', 'Failed to parse environment list')

        return []

    def install_package(self, env_name: str, package: str) -> bool:
        command = f"conda install -n {env_name} {package} -y"
        success = self.runner.run(command)[0]

        if success:
            self._update_registry(env_name, package)

        return success

    def update_package(self, env_name: str, package: str) -> bool:
        command = f"conda update -n {env_name} {package} -y"
        return self.runner.run(command)[0]

    def list_packages(self, env_name: str) -> List[Dict[str, str]]:
        command = f"conda list -n {env_name} --json"
        success, output = self.runner.run(command, live=False, capture=True)

        if success and output:
            try:
                packages = json.loads(output)
                return [{"name": pkg["name"], "version": pkg["version"]} for pkg in packages]
            except json.JSONDecodeError:
                print_status('error', 'Failed to parse package list')

        return []

    def run_script(self, env_name: str, script: str, args: List[str], python: bool = True) -> bool:
        if python:
            command = f"conda run -v --no-capture-output -n {env_name} python {script} {' '.join(args)}"
        else:
            command = f"conda run -v --no-capture-output -n {env_name} {script} {' '.join(args)}"

        return self.runner.run(command)[0]

    def _update_registry(self, env_name: str, package: str):
        """Update package registry."""
        registry_file = Path(f"{env_name}_registry.json")

        try:
            if registry_file.exists():
                with open(registry_file) as f:
                    registry = json.load(f)
            else:
                registry = []

            if package not in registry:
                registry.append(package)

            with open(registry_file, 'w') as f:
                json.dump(registry, f, indent=2)

            print_status('info', f'Updated registry: {registry_file}')

        except Exception as e:
            print_status('warning', f'Failed to update registry: {e}')

NativeManager

Native Python (venv + pip) manager implementation.

Source code in toolboxv2/utils/clis/venv_runner.py

class NativeManager(BasePackageManager):
    """Native Python (venv + pip) manager implementation."""

    def __init__(self, runner: CommandRunner):
        super().__init__(runner)
        self.envs_base = Path.home() / ".python_envs"
        self.envs_base.mkdir(exist_ok=True)

    def create_env(self, env_name: str, python_version: str = "3.11") -> bool:
        env_path = self.envs_base / env_name
        command = f"python{python_version} -m venv {env_path}"

        # Fallback to default python if version not available
        if not shutil.which(f"python{python_version}"):
            print_status('warning', f'Python {python_version} not found, using default python')
            command = f"python -m venv {env_path}"

        return self.runner.run(command)[0]

    def delete_env(self, env_name: str) -> bool:
        env_path = self.envs_base / env_name

        if env_path.exists():
            try:
                shutil.rmtree(env_path)
                print_status('success', f'Removed environment: {env_path}')
                return True
            except Exception as e:
                print_status('error', f'Failed to remove environment: {e}')
                return False
        else:
            print_status('warning', f'Environment not found: {env_path}')
            return False

    def list_envs(self) -> List[str]:
        if self.envs_base.exists():
            return [d.name for d in self.envs_base.iterdir() if d.is_dir() and (d / "bin" / "python").exists()]
        return []

    def install_package(self, env_name: str, package: str) -> bool:
        env_path = self.envs_base / env_name
        pip_bin = env_path / "bin" / "pip"

        if sys.platform == "win32":
            pip_bin = env_path / "Scripts" / "pip.exe"

        command = f"{pip_bin} install {package}"
        return self.runner.run(command)[0]

    def update_package(self, env_name: str, package: str) -> bool:
        env_path = self.envs_base / env_name
        pip_bin = env_path / "bin" / "pip"

        if sys.platform == "win32":
            pip_bin = env_path / "Scripts" / "pip.exe"

        command = f"{pip_bin} install --upgrade {package}"
        return self.runner.run(command)[0]

    def list_packages(self, env_name: str) -> List[Dict[str, str]]:
        env_path = self.envs_base / env_name
        pip_bin = env_path / "bin" / "pip"

        if sys.platform == "win32":
            pip_bin = env_path / "Scripts" / "pip.exe"

        command = f"{pip_bin} list --format json"
        success, output = self.runner.run(command, live=False, capture=True)

        if success and output:
            try:
                packages = json.loads(output)
                return [{"name": pkg["name"], "version": pkg["version"]} for pkg in packages]
            except json.JSONDecodeError:
                print_status('error', 'Failed to parse package list')

        return []

    def run_script(self, env_name: str, script: str, args: List[str], python: bool = True) -> bool:
        env_path = self.envs_base / env_name
        python_bin = env_path / "bin" / "python"

        if sys.platform == "win32":
            python_bin = env_path / "Scripts" / "python.exe"

        if python:
            command = f"{python_bin} {script} {' '.join(args)}"
        else:
            command = f"{script} {' '.join(args)}"

        return self.runner.run(command)[0]

PackageManager

Supported package managers.

Source code in toolboxv2/utils/clis/venv_runner.py

class PackageManager(Enum):
    """Supported package managers."""
    CONDA = "conda"
    UV = "uv"
    NATIVE = "native"  # pip/venv

UVManager

UV package manager implementation.

Source code in toolboxv2/utils/clis/venv_runner.py

class UVManager(BasePackageManager):
    """UV package manager implementation."""

    def create_env(self, env_name: str, python_version: str = "3.11") -> bool:
        env_path = Path.home() / ".uv" / "envs" / env_name
        command = f"uv venv {env_path} --python {python_version}"
        return self.runner.run(command)[0]

    def delete_env(self, env_name: str) -> bool:
        env_path = Path.home() / ".uv" / "envs" / env_name

        if env_path.exists():
            try:
                shutil.rmtree(env_path)
                print_status('success', f'Removed environment: {env_path}')
                return True
            except Exception as e:
                print_status('error', f'Failed to remove environment: {e}')
                return False
        else:
            print_status('warning', f'Environment not found: {env_path}')
            return False

    def list_envs(self) -> List[str]:
        envs_path = Path.home() / ".uv" / "envs"

        if envs_path.exists():
            return [d.name for d in envs_path.iterdir() if d.is_dir()]

        return []

    def install_package(self, env_name: str, package: str) -> bool:
        env_path = Path.home() / ".uv" / "envs" / env_name
        command = f"uv pip install --python {env_path}/bin/python {package}"
        return self.runner.run(command)[0]

    def update_package(self, env_name: str, package: str) -> bool:
        env_path = Path.home() / ".uv" / "envs" / env_name
        command = f"uv pip install --upgrade --python {env_path}/bin/python {package}"
        return self.runner.run(command)[0]

    def list_packages(self, env_name: str) -> List[Dict[str, str]]:
        env_path = Path.home() / ".uv" / "envs" / env_name
        command = f"uv pip list --python {env_path}/bin/python --format json"
        success, output = self.runner.run(command, live=False, capture=True)

        if success and output:
            try:
                packages = json.loads(output)
                return [{"name": pkg["name"], "version": pkg["version"]} for pkg in packages]
            except json.JSONDecodeError:
                print_status('error', 'Failed to parse package list')

        return []

    def run_script(self, env_name: str, script: str, args: List[str], python: bool = True) -> bool:
        env_path = Path.home() / ".uv" / "envs" / env_name
        python_bin = env_path / "bin" / "python"

        if python:
            command = f"{python_bin} {script} {' '.join(args)}"
        else:
            command = f"{script} {' '.join(args)}"

        return self.runner.run(command)[0]

create_manager(pm_type)

Create appropriate package manager.

Source code in toolboxv2/utils/clis/venv_runner.py

def create_manager(pm_type: PackageManager) -> BasePackageManager:
    """Create appropriate package manager."""
    runner = CommandRunner(pm_type)

    if pm_type == PackageManager.CONDA:
        return CondaManager(runner)
    elif pm_type == PackageManager.UV:
        return UVManager(runner)
    else:
        return NativeManager(runner)

create_parser()

Create modern CLI parser.

Source code in toolboxv2/utils/clis/venv_runner.py

def create_parser() -> argparse.ArgumentParser:
    """Create modern CLI parser."""

    parser = argparse.ArgumentParser(
        prog='tb venv',
        description=textwrap.dedent("""
        ╔════════════════════════════════════════════════════════════════════╗
        ║          🐍 Modern Python Environment Manager 🐍                   ║
        ╚════════════════════════════════════════════════════════════════════╝

        Unified interface for conda, uv, and native Python environments.

        """),
        epilog=textwrap.dedent("""
        ┌─ EXAMPLES ─────────────────────────────────────────────────────────┐
        │                                                                    │
        │  Environment Management:                                           │
        │    $ tb venv create myenv                  # Create environment    │
        │    $ tb venv list                          # List environments     │
        │    $ tb venv delete myenv                  # Delete environment    │
        │                                                                    │
        │  Package Management:                                               │
        │    $ tb venv install myenv numpy           # Install package       │
        │    $ tb venv update myenv numpy            # Update package        │
        │    $ tb venv packages myenv                # List packages         │
        │                                                                    │
        │  Script Execution:                                                 │
        │    $ tb venv run myenv script.py arg1      # Run Python script     │
        │    $ tb venv exec myenv command args       # Run command           │
        │                                                                    │
        │  Advanced:                                                         │
        │    $ tb venv registry myenv                # Create registry       │
        │    $ tb venv update-all myenv              # Update all packages   │
        │    $ tb venv --manager uv create myenv     # Use specific PM       │
        │                                                                    │
        └────────────────────────────────────────────────────────────────────┘
        """),
        formatter_class=argparse.RawDescriptionHelpFormatter
    )

    # Global options
    parser.add_argument('--manager', '-m',
                        choices=['conda', 'uv', 'native'],
                        help='Package manager to use (auto-detect if not specified)')

    parser.add_argument('--python', '-py',
                        default='3.11',
                        help='Python version (default: 3.11)')

    # Subcommands
    subparsers = parser.add_subparsers(dest='command', help='Available commands')

    # =================== ENVIRONMENT COMMANDS ===================

    # Create environment
    create_parser = subparsers.add_parser('create', help='Create new environment')
    create_parser.add_argument('env_name', help='Environment name')
    create_parser.add_argument('--python', '-py', help='Python version (default: 3.11)')

    # Delete environment
    delete_parser = subparsers.add_parser('delete', help='Delete environment')
    delete_parser.add_argument('env_name', help='Environment name')
    delete_parser.add_argument('--force', '-f', action='store_true', help='Skip confirmation')

    # List environments
    list_parser = subparsers.add_parser('list', help='List all environments')

    # =================== PACKAGE COMMANDS ===================

    # Install package
    install_parser = subparsers.add_parser('install', help='Install package')
    install_parser.add_argument('env_name', help='Environment name')
    install_parser.add_argument('packages', nargs='+', help='Package(s) to install')
    install_parser.add_argument('--save', '-s', action='store_true', help='Save to registry')

    # Update package
    update_parser = subparsers.add_parser('update', help='Update package')
    update_parser.add_argument('env_name', help='Environment name')
    update_parser.add_argument('package', nargs='?', help='Package to update (all if not specified)')

    # List packages
    packages_parser = subparsers.add_parser('packages', help='List installed packages')
    packages_parser.add_argument('env_name', help='Environment name')
    packages_parser.add_argument('--json', action='store_true', help='Output as JSON')

    # =================== EXECUTION COMMANDS ===================

    # Run Python script
    run_parser = subparsers.add_parser('run', help='Run Python script in environment')
    run_parser.add_argument('env_name', help='Environment name')
    run_parser.add_argument('script', help='Script to run')
    run_parser.add_argument('args', nargs='*', help='Script arguments')

    # Execute command
    exec_parser = subparsers.add_parser('exec', help='Execute command in environment')
    exec_parser.add_argument('env_name', help='Environment name')
    exec_parser.add_argument('command', help='Command to execute')
    exec_parser.add_argument('args', nargs='*', help='Command arguments')

    # =================== UTILITY COMMANDS ===================

    # Create registry
    registry_parser = subparsers.add_parser('registry', help='Create package registry')
    registry_parser.add_argument('env_name', help='Environment name')

    # Update all packages
    update_all_parser = subparsers.add_parser('update-all', help='Update all packages')
    update_all_parser.add_argument('env_name', help='Environment name')

    # Info command
    info_parser = subparsers.add_parser('info', help='Show environment information')
    info_parser.add_argument('env_name', nargs='?', help='Environment name (current if not specified)')
    # Discover environments
    discover_parser = subparsers.add_parser('discover', help='Discover existing environments from all managers')
    discover_parser.add_argument('--save', '-s', action='store_true', help='Save discovered environments to registry')
    discover_parser.add_argument('--json', action='store_true', help='Output as JSON')
    return parser

detect_package_manager()

Auto-detect available package manager.

Source code in toolboxv2/utils/clis/venv_runner.py

def detect_package_manager() -> PackageManager:
    """Auto-detect available package manager."""
    if shutil.which("uv"):
        return PackageManager.UV
    elif shutil.which("conda"):
        return PackageManager.CONDA
    else:
        return PackageManager.NATIVE

discover_environments()

Discover existing environments from all package managers.

Source code in toolboxv2/utils/clis/venv_runner.py

def discover_environments() -> Dict[str, List[Dict[str, str]]]:
    """Discover existing environments from all package managers."""
    from toolboxv2 import init_cwd
    discovered = {
        'conda': [],
        'uv': [],
        'native': []
    }

    # Discover Conda environments
    if shutil.which("conda"):
        try:
            result = subprocess.run(
                ["conda", "env", "list", "--json"],
                capture_output=True, text=True, check=True
            )
            data = json.loads(result.stdout)
            for env_path in data.get('envs', []):
                env_name = Path(env_path).name
                if env_name != 'base':  # Skip base environment
                    discovered['conda'].append({
                        'name': env_name,
                        'path': env_path,
                        'manager': 'conda'
                    })
        except (subprocess.CalledProcessError, json.JSONDecodeError):
            pass

    # Discover UV environments
    if shutil.which("uv"):
        uv_envs_path = Path.home() / ".uv" / "envs"
        if uv_envs_path.exists():
            for env_dir in uv_envs_path.iterdir():
                if env_dir.is_dir():
                    discovered['uv'].append({
                        'name': env_dir.name,
                        'path': str(env_dir),
                        'manager': 'uv'
                    })

    # Discover Native Python environments
    native_paths = [
        Path.home() / "python_env",
        Path.home() / ".python_envs",
        Path.cwd() / "venv",
        Path.cwd() / ".venv",
        Path.cwd() / "env",
        init_cwd / "python_env",
        init_cwd / ".python_envs",
        init_cwd/ "venv",
        init_cwd/ ".venv",
        init_cwd/ "env"
    ]

    for base_path in native_paths:
        if base_path.exists():
            if base_path.name in ['venv', '.venv', 'env']:
                # Single environment in current directory
                if _is_valid_venv(base_path):
                    discovered['native'].append({
                        'name': f"local-{base_path.name}",
                        'path': str(base_path),
                        'manager': 'native'
                    })
            else:
                # Multiple environments in directory
                for env_dir in base_path.iterdir():
                    if env_dir.is_dir() and _is_valid_venv(env_dir):
                        discovered['native'].append({
                            'name': env_dir.name,
                            'path': str(env_dir),
                            'manager': 'native'
                        })

    return discovered

get_encoding()

Get system encoding with fallback.

Source code in toolboxv2/utils/clis/venv_runner.py

def get_encoding():
    """Get system encoding with fallback."""
    try:
        return sys.stdout.encoding or 'utf-8'
    except:
        return 'utf-8'

handle_create(args, manager)

Handle environment creation.

Source code in toolboxv2/utils/clis/venv_runner.py

def handle_create(args, manager: BasePackageManager):
    """Handle environment creation."""
    print_header(f'Creating Environment: {args.env_name}')

    python_version = args.python or "3.11"

    if manager.create_env(args.env_name, python_version):
        print_status('success', f'Environment "{args.env_name}" created successfully!')
    else:
        print_status('error', f'Failed to create environment "{args.env_name}"')
        sys.exit(1)

handle_delete(args, manager)

Handle environment deletion.

Source code in toolboxv2/utils/clis/venv_runner.py

def handle_delete(args, manager: BasePackageManager):
    """Handle environment deletion."""
    if not args.force:
        # Confirm deletion
        result = yes_no_dialog(
            title='Confirm Deletion',
            text=f'Really delete environment "{args.env_name}"?\n\nThis action cannot be undone.',
            style=MODERN_STYLE
        ).run()

        if not result:
            print_status('info', 'Deletion cancelled')
            return

    print_header(f'Deleting Environment: {args.env_name}')

    if manager.delete_env(args.env_name):
        print_status('success', f'Environment "{args.env_name}" deleted successfully!')
    else:
        print_status('error', f'Failed to delete environment "{args.env_name}"')
        sys.exit(1)

handle_discover(args, manager)

Handle environment discovery.

Source code in toolboxv2/utils/clis/venv_runner.py

def handle_discover(args, manager: BasePackageManager):
    """Handle environment discovery."""
    print_header('Discovering Environments')

    discovered = discover_environments()

    total_found = sum(len(envs) for envs in discovered.values())

    if total_found == 0:
        print_status('warning', 'No environments discovered')
        return

    if args.json:
        print(json.dumps(discovered, indent=2))
        return

    # Display discovered environments
    for manager_name, envs in discovered.items():
        if envs:
            print(f"\n📦 {manager_name.upper()} Environments ({len(envs)} found):")
            print('─' * 60)

            for i, env in enumerate(envs, 1):
                print(f"  {i:>2}. {env['name']:<25} → {env['path']}")

    print(f"\n🔍 Total discovered: {total_found} environment(s)")

    # Save to registry if requested
    if args.save:
        try:
            registry_file = save_discovered_environments(discovered)
            print_status('success', f'Environments saved to registry: {registry_file}')
        except Exception as e:
            print_status('error', f'Failed to save registry: {e}')

handle_exec(args, manager)

Handle command execution.

Source code in toolboxv2/utils/clis/venv_runner.py

def handle_exec(args, manager: BasePackageManager):
    """Handle command execution."""
    print_header(f'Executing Command in: {args.env_name}')

    if manager.run_script(args.env_name, args.command, args.args, python=False):
        print_status('success', 'Command completed successfully')
    else:
        print_status('error', 'Command execution failed')
        sys.exit(1)

handle_info(args, manager)

Handle info display.

Source code in toolboxv2/utils/clis/venv_runner.py

def handle_info(args, manager: BasePackageManager):
    """Handle info display."""
    env_name = args.env_name or 'current'

    print_header(f'Environment Info: {env_name}')

    # Show package count
    packages = manager.list_packages(env_name) if args.env_name else []

    print(f"Package Manager: {manager.runner.pm.value}")
    print(f"Total Packages: {len(packages)}")
    print()

handle_install(args, manager)

Handle package installation.

Source code in toolboxv2/utils/clis/venv_runner.py

def handle_install(args, manager: BasePackageManager):
    """Handle package installation."""
    print_header(f'Installing Packages in: {args.env_name}')

    for package in args.packages:
        print(f"\nInstalling {package}...")

        if manager.install_package(args.env_name, package):
            print_status('success', f'Package "{package}" installed')
        else:
            print_status('error', f'Failed to install "{package}"')

handle_list(args, manager)

Handle environment listing.

Source code in toolboxv2/utils/clis/venv_runner.py

def handle_list(args, manager: BasePackageManager):
    """Handle environment listing."""
    print_header('Available Environments')

    envs = manager.list_envs()

    if not envs:
        print_status('warning', 'No environments found')
        return

    print(f"\n{'#':<4} {'Environment Name':<30}")
    print('─' * 50)

    for i, env in enumerate(envs, 1):
        print(f"{i:<4} {env:<30}")

    print(f"\nTotal: {len(envs)} environment(s)\n")

handle_packages(args, manager)

Handle package listing.

Source code in toolboxv2/utils/clis/venv_runner.py

def handle_packages(args, manager: BasePackageManager):
    """Handle package listing."""
    print_header(f'Packages in: {args.env_name}')

    packages = manager.list_packages(args.env_name)

    if not packages:
        print_status('warning', 'No packages found')
        return

    if args.json:
        print(json.dumps(packages, indent=2))
    else:
        print(f"\n{'#':<6} {'Package':<35} {'Version':<15}")
        print('─' * 60)

        for i, pkg in enumerate(packages, 1):
            print(f"{i:<6} {pkg['name']:<35} {pkg['version']:<15}")

        print(f"\nTotal: {len(packages)} package(s)\n")

handle_registry(args, manager)

Handle registry creation.

Source code in toolboxv2/utils/clis/venv_runner.py

def handle_registry(args, manager: BasePackageManager):
    """Handle registry creation."""
    print_header(f'Creating Registry for: {args.env_name}')

    packages = manager.list_packages(args.env_name)

    if not packages:
        print_status('warning', 'No packages to register')
        return

    registry_file = Path(f"{args.env_name}_registry.json")

    try:
        with open(registry_file, 'w') as f:
            json.dump(packages, f, indent=2)

        print_status('success', f'Registry created: {registry_file}')
        print_status('info', f'Registered {len(packages)} package(s)')

    except Exception as e:
        print_status('error', f'Failed to create registry: {e}')
        sys.exit(1)

handle_run(args, manager)

Handle script execution.

Source code in toolboxv2/utils/clis/venv_runner.py

def handle_run(args, manager: BasePackageManager):
    """Handle script execution."""
    print_header(f'Running Script in: {args.env_name}')

    if manager.run_script(args.env_name, args.script, args.args, python=True):
        print_status('success', 'Script completed successfully')
    else:
        print_status('error', 'Script execution failed')
        sys.exit(1)

handle_update(args, manager)

Handle package update.

Source code in toolboxv2/utils/clis/venv_runner.py

def handle_update(args, manager: BasePackageManager):
    """Handle package update."""
    print_header(f'Updating Packages in: {args.env_name}')

    if args.package:
        # Update single package
        if manager.update_package(args.env_name, args.package):
            print_status('success', f'Package "{args.package}" updated')
        else:
            print_status('error', f'Failed to update "{args.package}"')
    else:
        # Update all packages
        packages = manager.list_packages(args.env_name)

        if not packages:
            print_status('warning', 'No packages found')
            return

        print(f"Updating {len(packages)} package(s)...")

        for pkg in tqdm(packages, desc="Updating"):
            manager.update_package(args.env_name, pkg['name'])

        print_status('success', 'All packages updated')

main()

Main entry point.

Source code in toolboxv2/utils/clis/venv_runner.py

def main():
    """Main entry point."""
    parser = create_parser()
    args = parser.parse_args()

    if not args.command:
        parser.print_help()
        return

    # Determine package manager
    if args.manager:
        pm_type = PackageManager(args.manager)
    else:
        pm_type = detect_package_manager()
        print_status('info', f'Auto-detected package manager: {pm_type.value}')

    # Create manager
    manager = create_manager(pm_type)

    # Handle command
    try:
        if args.command == 'create':
            handle_create(args, manager)
        elif args.command == 'delete':
            handle_delete(args, manager)
        elif args.command == 'list':
            handle_list(args, manager)
        elif args.command == 'install':
            handle_install(args, manager)
        elif args.command == 'update':
            handle_update(args, manager)
        elif args.command == 'packages':
            handle_packages(args, manager)
        elif args.command == 'run':
            handle_run(args, manager)
        elif args.command == 'exec':
            handle_exec(args, manager)
        elif args.command == 'registry':
            handle_registry(args, manager)
        elif args.command == 'update-all':
            handle_update(args, manager)
        elif args.command == 'info':
            handle_info(args, manager)
        elif args.command == 'discover':
            handle_discover(args, manager)

    except KeyboardInterrupt:
        print_status('warning', '\nOperation cancelled by user')
        sys.exit(130)

    except Exception as e:
        print_status('error', f'Unexpected error: {e}')
        import traceback
        traceback.print_exc()
        sys.exit(1)

print_header(title)

Print section header.

Source code in toolboxv2/utils/clis/venv_runner.py

def print_header(title: str):
    """Print section header."""
    width = 78
    print_formatted_text(HTML(f'\n<header>{"─" * width}</header>'))
    print_formatted_text(HTML(f'<header>{title.center(width)}</header>'))
    print_formatted_text(HTML(f'<header>{"─" * width}</header>\n'))

print_status(status, message)

Print colored status message.

Source code in toolboxv2/utils/clis/venv_runner.py

def print_status(status: str, message: str):
    """Print colored status message."""
    icons = {
        'success': '✓',
        'error': '✗',
        'warning': '⚠',
        'info': 'ℹ',
        'running': '⟳'
    }
    icon = icons.get(status, '•')
    print_formatted_text(HTML(f'<{status}>{icon} {message}</{status}>'), style=MODERN_STYLE)

save_discovered_environments(discovered)

Save discovered environments to registry file.

Source code in toolboxv2/utils/clis/venv_runner.py

def save_discovered_environments(discovered: Dict[str, List[Dict[str, str]]]) -> Path:
    """Save discovered environments to registry file."""
    registry_file = Path.home() / ".toolbox_env_registry.json"

    # Load existing registry or create new
    existing_registry = {}
    if registry_file.exists():
        try:
            with open(registry_file, 'r') as f:
                existing_registry = json.load(f)
        except (json.JSONDecodeError, IOError):
            pass

    # Merge discovered environments
    for manager, envs in discovered.items():
        if manager not in existing_registry:
            existing_registry[manager] = []

        # Add new environments (avoid duplicates)
        existing_names = {env['name'] for env in existing_registry[manager]}
        for env in envs:
            if env['name'] not in existing_names:
                existing_registry[manager].append(env)

    # Save updated registry
    try:
        with open(registry_file, 'w') as f:
            json.dump(existing_registry, f, indent=2)
        return registry_file
    except IOError as e:
        raise Exception(f"Failed to save registry: {e}")

daemon

DaemonUtil

Source code in toolboxv2/utils/daemon/daemon_util.py

class DaemonUtil:

    def __init__(self, *args, **kwargs):
        """
        Standard constructor used for arguments pass
        Do not override. Use __ainit__ instead
        """
        self.server = None
        self.alive = False
        self.__storedargs = args, kwargs
        self.async_initialized = False

    async def __initobj(self):
        """Crutch used for __await__ after spawning"""
        assert not self.async_initialized
        self.async_initialized = True
        # pass the parameters to __ainit__ that passed to __init__
        await self.__ainit__(*self.__storedargs[0], **self.__storedargs[1])
        return self

    def __await__(self):
        return self.__initobj().__await__()

    async def __ainit__(self, class_instance: Any, host='0.0.0.0', port=6587, t=False,
                        app: (App or AppType) | None = None,
                        peer=False, name='daemonApp-server', on_register=None, on_client_exit=None, on_server_exit=None,
                        unix_socket=False, test_override=False):
        from toolboxv2.mods.SocketManager import SocketType
        self.class_instance = class_instance
        self.server = None
        self.port = port
        self.host = host
        self.alive = False
        self.test_override = test_override
        self._name = name
        if on_register is None:
            def on_register(*args):
                return None
        self._on_register = on_register
        if on_client_exit is None:
            def on_client_exit(*args):
                return None
        self.on_client_exit = on_client_exit
        if on_server_exit is None:
            def on_server_exit():
                return None
        self.on_server_exit = on_server_exit
        self.unix_socket = unix_socket
        self.online = None
        connection_type = SocketType.server
        if peer:
            connection_type = SocketType.peer

        await self.start_server(connection_type)
        app = app if app is not None else get_app(from_=f"DaemonUtil.{self._name}")
        self.online = await asyncio.to_thread(self.connect, app)
        if t:
            await self.online

    async def start_server(self, connection_type=None):
        """Start the server using app and the socket manager"""
        from toolboxv2.mods.SocketManager import SocketType
        if connection_type is None:
            connection_type = SocketType.server
        app = get_app(from_="Starting.Daemon")
        print(app.mod_online("SocketManager"), "SocketManager")
        if not app.mod_online("SocketManager"):
            await app.load_mod("SocketManager")
        server_result = await app.a_run_any(SOCKETMANAGER.CREATE_SOCKET,
                                            get_results=True,
                                            name=self._name,
                                            host=self.host,
                                            port=self.port,
                                            type_id=connection_type,
                                            max_connections=-1,
                                            return_full_object=True,
                                            test_override=self.test_override,
                                            unix_file=self.unix_socket)
        if server_result.is_error():
            raise Exception(f"Server error: {server_result.print(False)}")
        if not server_result.is_data():
            raise Exception(f"Server error: {server_result.print(False)}")
        self.alive = True
        self.server = server_result
        # 'socket': socket,
        # 'receiver_socket': r_socket,
        # 'host': host,
        # 'port': port,
        # 'p2p-port': endpoint_port,
        # 'sender': send,
        # 'receiver_queue': receiver_queue,
        # 'connection_error': connection_error,
        # 'receiver_thread': s_thread,
        # 'keepalive_thread': keep_alive_thread,
        # 'running_dict': running_dict,
        # 'client_to_receiver_thread': to_receive,
        # 'client_receiver_threads': threeds,

    async def send(self, data: dict or bytes or str, identifier: tuple[str, int] or str = "main"):
        result = await self.server.aget()
        sender = result.get('sender')
        await sender(data, identifier)
        return "Data Transmitted"

    @staticmethod
    async def runner_co(fuction, *args, **kwargs):
        if asyncio.iscoroutinefunction(fuction):
            return await fuction(*args, **kwargs)
        return fuction(*args, **kwargs)

    async def connect(self, app):
        result = await self.server.aget()
        if not isinstance(result, dict) or result.get('connection_error') != 0:
            raise Exception(f"Server error: {result}")
        self.server = Result.ok(result)
        receiver_queue: queue.Queue = self.server.get('receiver_queue')
        client_to_receiver_thread = self.server.get('client_to_receiver_thread')
        running_dict = self.server.get('running_dict')
        sender = self.server.get('sender')
        known_clients = {}
        valid_clients = {}
        app.print(f"Starting Demon {self._name}")

        while self.alive:

            if not receiver_queue.empty():
                data = receiver_queue.get()
                print(data)
                if not data:
                    continue
                if 'identifier' not in data:
                    continue

                identifier = data.get('identifier', 'unknown')
                try:
                    if identifier == "new_con":
                        client, address = data.get('data')
                        get_logger().info(f"New connection: {address}")
                        known_clients[str(address)] = client
                        await client_to_receiver_thread(client, str(address))

                        await self.runner_co(self._on_register, identifier, address)
                        identifier = str(address)
                        # await sender({'ok': 0}, identifier)

                    print("Receiver queue", identifier, identifier in known_clients, identifier in valid_clients)
                    # validation
                    if identifier in known_clients:
                        get_logger().info(identifier)
                        if identifier.startswith("('127.0.0.1'"):
                            valid_clients[identifier] = known_clients[identifier]
                            await self.runner_co(self._on_register, identifier, data)
                        elif data.get("claim", False):
                            do = app.run_any(("CloudM.UserInstances", "validate_ws_id"),
                                             ws_id=data.get("claim"))[0]
                            get_logger().info(do)
                            if do:
                                valid_clients[identifier] = known_clients[identifier]
                                await self.runner_co(self._on_register, identifier, data)
                        elif data.get("key", False) == os.getenv("TB_R_KEY"):
                            valid_clients[identifier] = known_clients[identifier]
                            await self.runner_co(self._on_register, identifier, data)
                        else:
                            get_logger().warning(f"Validating Failed: {identifier}")
                            # sender({'Validating Failed': -1}, eval(identifier))
                        get_logger().info(f"Validating New: {identifier}")
                        del known_clients[identifier]

                    elif identifier in valid_clients:
                        get_logger().info(f"New valid Request: {identifier}")
                        name = data.get('name')
                        args = data.get('args')
                        kwargs = data.get('kwargs')
                        if not name:
                            continue

                        get_logger().info(f"Request data: {name=}{args=}{kwargs=}{identifier=}")

                        if name == 'exit_main':
                            self.alive = False
                            break

                        if name == 'show_console':
                            show_console(True)
                            await sender({'ok': 0}, identifier)
                            continue

                        if name == 'hide_console':
                            show_console(False)
                            await sender({'ok': 0}, identifier)
                            continue

                        if name == 'rrun_flow':
                            show_console(True)
                            runnner = self.class_instance.run_flow
                            threading.Thread(target=runnner, args=args, kwargs=kwargs, daemon=True).start()
                            await sender({'ok': 0}, identifier)
                            show_console(False)
                            continue

                        async def _helper_runner():
                            try:
                                attr_f = getattr(self.class_instance, name)

                                if asyncio.iscoroutinefunction(attr_f):
                                    res = await attr_f(*args, **kwargs)
                                else:
                                    res = attr_f(*args, **kwargs)

                                if res is None:
                                    res = {'data': res}
                                elif isinstance(res, Result):
                                    if asyncio.iscoroutine(res.get()) or isinstance(res.get(), asyncio.Task):
                                        res_ = await res.aget()
                                        res.result.data = res_
                                    res = json.loads(res.to_api_result().json())
                                elif isinstance(res, bytes | dict):
                                    pass
                                else:
                                    res = {'data': 'unsupported type', 'type': str(type(res))}

                                get_logger().info(f"sending response {res} {type(res)}")

                                await sender(res, identifier)
                            except Exception as e:
                                import traceback
                                print(traceback.format_exc())
                                await sender({"data": str(e)}, identifier)

                        await _helper_runner()
                    else:
                        print("Unknown connection data:", data)

                except Exception as e:
                    get_logger().warning(Style.RED(f"An error occurred on {identifier} {str(e)}"))
                    if identifier != "unknown":
                        running_dict["receive"][str(identifier)] = False
                        await self.runner_co(self.on_client_exit,  identifier)
            await asyncio.sleep(0.1)
        running_dict["server_receiver"] = False
        for x in running_dict["receive"]:
            running_dict["receive"][x] = False
        running_dict["keep_alive_var"] = False
        await self.runner_co(self.on_server_exit)
        app.print(f"Closing Demon {self._name}")
        return Result.ok()

    async def a_exit(self):
        result = await self.server.aget()
        await result.get("close")()
        self.alive = False
        if asyncio.iscoroutine(self.online):
            await self.online
        print("Connection result :", result.get("host"), result.get("port"),
              "total connections:", result.get("connections"))

__init__(*args, **kwargs)

Standard constructor used for arguments pass Do not override. Use ainit instead

Source code in toolboxv2/utils/daemon/daemon_util.py

def __init__(self, *args, **kwargs):
    """
    Standard constructor used for arguments pass
    Do not override. Use __ainit__ instead
    """
    self.server = None
    self.alive = False
    self.__storedargs = args, kwargs
    self.async_initialized = False

__initobj() async

Crutch used for await after spawning

Source code in toolboxv2/utils/daemon/daemon_util.py

async def __initobj(self):
    """Crutch used for __await__ after spawning"""
    assert not self.async_initialized
    self.async_initialized = True
    # pass the parameters to __ainit__ that passed to __init__
    await self.__ainit__(*self.__storedargs[0], **self.__storedargs[1])
    return self

start_server(connection_type=None) async

Start the server using app and the socket manager

Source code in toolboxv2/utils/daemon/daemon_util.py

async def start_server(self, connection_type=None):
    """Start the server using app and the socket manager"""
    from toolboxv2.mods.SocketManager import SocketType
    if connection_type is None:
        connection_type = SocketType.server
    app = get_app(from_="Starting.Daemon")
    print(app.mod_online("SocketManager"), "SocketManager")
    if not app.mod_online("SocketManager"):
        await app.load_mod("SocketManager")
    server_result = await app.a_run_any(SOCKETMANAGER.CREATE_SOCKET,
                                        get_results=True,
                                        name=self._name,
                                        host=self.host,
                                        port=self.port,
                                        type_id=connection_type,
                                        max_connections=-1,
                                        return_full_object=True,
                                        test_override=self.test_override,
                                        unix_file=self.unix_socket)
    if server_result.is_error():
        raise Exception(f"Server error: {server_result.print(False)}")
    if not server_result.is_data():
        raise Exception(f"Server error: {server_result.print(False)}")
    self.alive = True
    self.server = server_result

daemon_util

DaemonUtil

Source code in toolboxv2/utils/daemon/daemon_util.py

class DaemonUtil:

    def __init__(self, *args, **kwargs):
        """
        Standard constructor used for arguments pass
        Do not override. Use __ainit__ instead
        """
        self.server = None
        self.alive = False
        self.__storedargs = args, kwargs
        self.async_initialized = False

    async def __initobj(self):
        """Crutch used for __await__ after spawning"""
        assert not self.async_initialized
        self.async_initialized = True
        # pass the parameters to __ainit__ that passed to __init__
        await self.__ainit__(*self.__storedargs[0], **self.__storedargs[1])
        return self

    def __await__(self):
        return self.__initobj().__await__()

    async def __ainit__(self, class_instance: Any, host='0.0.0.0', port=6587, t=False,
                        app: (App or AppType) | None = None,
                        peer=False, name='daemonApp-server', on_register=None, on_client_exit=None, on_server_exit=None,
                        unix_socket=False, test_override=False):
        from toolboxv2.mods.SocketManager import SocketType
        self.class_instance = class_instance
        self.server = None
        self.port = port
        self.host = host
        self.alive = False
        self.test_override = test_override
        self._name = name
        if on_register is None:
            def on_register(*args):
                return None
        self._on_register = on_register
        if on_client_exit is None:
            def on_client_exit(*args):
                return None
        self.on_client_exit = on_client_exit
        if on_server_exit is None:
            def on_server_exit():
                return None
        self.on_server_exit = on_server_exit
        self.unix_socket = unix_socket
        self.online = None
        connection_type = SocketType.server
        if peer:
            connection_type = SocketType.peer

        await self.start_server(connection_type)
        app = app if app is not None else get_app(from_=f"DaemonUtil.{self._name}")
        self.online = await asyncio.to_thread(self.connect, app)
        if t:
            await self.online

    async def start_server(self, connection_type=None):
        """Start the server using app and the socket manager"""
        from toolboxv2.mods.SocketManager import SocketType
        if connection_type is None:
            connection_type = SocketType.server
        app = get_app(from_="Starting.Daemon")
        print(app.mod_online("SocketManager"), "SocketManager")
        if not app.mod_online("SocketManager"):
            await app.load_mod("SocketManager")
        server_result = await app.a_run_any(SOCKETMANAGER.CREATE_SOCKET,
                                            get_results=True,
                                            name=self._name,
                                            host=self.host,
                                            port=self.port,
                                            type_id=connection_type,
                                            max_connections=-1,
                                            return_full_object=True,
                                            test_override=self.test_override,
                                            unix_file=self.unix_socket)
        if server_result.is_error():
            raise Exception(f"Server error: {server_result.print(False)}")
        if not server_result.is_data():
            raise Exception(f"Server error: {server_result.print(False)}")
        self.alive = True
        self.server = server_result
        # 'socket': socket,
        # 'receiver_socket': r_socket,
        # 'host': host,
        # 'port': port,
        # 'p2p-port': endpoint_port,
        # 'sender': send,
        # 'receiver_queue': receiver_queue,
        # 'connection_error': connection_error,
        # 'receiver_thread': s_thread,
        # 'keepalive_thread': keep_alive_thread,
        # 'running_dict': running_dict,
        # 'client_to_receiver_thread': to_receive,
        # 'client_receiver_threads': threeds,

    async def send(self, data: dict or bytes or str, identifier: tuple[str, int] or str = "main"):
        result = await self.server.aget()
        sender = result.get('sender')
        await sender(data, identifier)
        return "Data Transmitted"

    @staticmethod
    async def runner_co(fuction, *args, **kwargs):
        if asyncio.iscoroutinefunction(fuction):
            return await fuction(*args, **kwargs)
        return fuction(*args, **kwargs)

    async def connect(self, app):
        result = await self.server.aget()
        if not isinstance(result, dict) or result.get('connection_error') != 0:
            raise Exception(f"Server error: {result}")
        self.server = Result.ok(result)
        receiver_queue: queue.Queue = self.server.get('receiver_queue')
        client_to_receiver_thread = self.server.get('client_to_receiver_thread')
        running_dict = self.server.get('running_dict')
        sender = self.server.get('sender')
        known_clients = {}
        valid_clients = {}
        app.print(f"Starting Demon {self._name}")

        while self.alive:

            if not receiver_queue.empty():
                data = receiver_queue.get()
                print(data)
                if not data:
                    continue
                if 'identifier' not in data:
                    continue

                identifier = data.get('identifier', 'unknown')
                try:
                    if identifier == "new_con":
                        client, address = data.get('data')
                        get_logger().info(f"New connection: {address}")
                        known_clients[str(address)] = client
                        await client_to_receiver_thread(client, str(address))

                        await self.runner_co(self._on_register, identifier, address)
                        identifier = str(address)
                        # await sender({'ok': 0}, identifier)

                    print("Receiver queue", identifier, identifier in known_clients, identifier in valid_clients)
                    # validation
                    if identifier in known_clients:
                        get_logger().info(identifier)
                        if identifier.startswith("('127.0.0.1'"):
                            valid_clients[identifier] = known_clients[identifier]
                            await self.runner_co(self._on_register, identifier, data)
                        elif data.get("claim", False):
                            do = app.run_any(("CloudM.UserInstances", "validate_ws_id"),
                                             ws_id=data.get("claim"))[0]
                            get_logger().info(do)
                            if do:
                                valid_clients[identifier] = known_clients[identifier]
                                await self.runner_co(self._on_register, identifier, data)
                        elif data.get("key", False) == os.getenv("TB_R_KEY"):
                            valid_clients[identifier] = known_clients[identifier]
                            await self.runner_co(self._on_register, identifier, data)
                        else:
                            get_logger().warning(f"Validating Failed: {identifier}")
                            # sender({'Validating Failed': -1}, eval(identifier))
                        get_logger().info(f"Validating New: {identifier}")
                        del known_clients[identifier]

                    elif identifier in valid_clients:
                        get_logger().info(f"New valid Request: {identifier}")
                        name = data.get('name')
                        args = data.get('args')
                        kwargs = data.get('kwargs')
                        if not name:
                            continue

                        get_logger().info(f"Request data: {name=}{args=}{kwargs=}{identifier=}")

                        if name == 'exit_main':
                            self.alive = False
                            break

                        if name == 'show_console':
                            show_console(True)
                            await sender({'ok': 0}, identifier)
                            continue

                        if name == 'hide_console':
                            show_console(False)
                            await sender({'ok': 0}, identifier)
                            continue

                        if name == 'rrun_flow':
                            show_console(True)
                            runnner = self.class_instance.run_flow
                            threading.Thread(target=runnner, args=args, kwargs=kwargs, daemon=True).start()
                            await sender({'ok': 0}, identifier)
                            show_console(False)
                            continue

                        async def _helper_runner():
                            try:
                                attr_f = getattr(self.class_instance, name)

                                if asyncio.iscoroutinefunction(attr_f):
                                    res = await attr_f(*args, **kwargs)
                                else:
                                    res = attr_f(*args, **kwargs)

                                if res is None:
                                    res = {'data': res}
                                elif isinstance(res, Result):
                                    if asyncio.iscoroutine(res.get()) or isinstance(res.get(), asyncio.Task):
                                        res_ = await res.aget()
                                        res.result.data = res_
                                    res = json.loads(res.to_api_result().json())
                                elif isinstance(res, bytes | dict):
                                    pass
                                else:
                                    res = {'data': 'unsupported type', 'type': str(type(res))}

                                get_logger().info(f"sending response {res} {type(res)}")

                                await sender(res, identifier)
                            except Exception as e:
                                import traceback
                                print(traceback.format_exc())
                                await sender({"data": str(e)}, identifier)

                        await _helper_runner()
                    else:
                        print("Unknown connection data:", data)

                except Exception as e:
                    get_logger().warning(Style.RED(f"An error occurred on {identifier} {str(e)}"))
                    if identifier != "unknown":
                        running_dict["receive"][str(identifier)] = False
                        await self.runner_co(self.on_client_exit,  identifier)
            await asyncio.sleep(0.1)
        running_dict["server_receiver"] = False
        for x in running_dict["receive"]:
            running_dict["receive"][x] = False
        running_dict["keep_alive_var"] = False
        await self.runner_co(self.on_server_exit)
        app.print(f"Closing Demon {self._name}")
        return Result.ok()

    async def a_exit(self):
        result = await self.server.aget()
        await result.get("close")()
        self.alive = False
        if asyncio.iscoroutine(self.online):
            await self.online
        print("Connection result :", result.get("host"), result.get("port"),
              "total connections:", result.get("connections"))

__init__(*args, **kwargs)

Standard constructor used for arguments pass Do not override. Use ainit instead

Source code in toolboxv2/utils/daemon/daemon_util.py

def __init__(self, *args, **kwargs):
    """
    Standard constructor used for arguments pass
    Do not override. Use __ainit__ instead
    """
    self.server = None
    self.alive = False
    self.__storedargs = args, kwargs
    self.async_initialized = False

__initobj() async

Crutch used for await after spawning

Source code in toolboxv2/utils/daemon/daemon_util.py

async def __initobj(self):
    """Crutch used for __await__ after spawning"""
    assert not self.async_initialized
    self.async_initialized = True
    # pass the parameters to __ainit__ that passed to __init__
    await self.__ainit__(*self.__storedargs[0], **self.__storedargs[1])
    return self

start_server(connection_type=None) async

Start the server using app and the socket manager

Source code in toolboxv2/utils/daemon/daemon_util.py

async def start_server(self, connection_type=None):
    """Start the server using app and the socket manager"""
    from toolboxv2.mods.SocketManager import SocketType
    if connection_type is None:
        connection_type = SocketType.server
    app = get_app(from_="Starting.Daemon")
    print(app.mod_online("SocketManager"), "SocketManager")
    if not app.mod_online("SocketManager"):
        await app.load_mod("SocketManager")
    server_result = await app.a_run_any(SOCKETMANAGER.CREATE_SOCKET,
                                        get_results=True,
                                        name=self._name,
                                        host=self.host,
                                        port=self.port,
                                        type_id=connection_type,
                                        max_connections=-1,
                                        return_full_object=True,
                                        test_override=self.test_override,
                                        unix_file=self.unix_socket)
    if server_result.is_error():
        raise Exception(f"Server error: {server_result.print(False)}")
    if not server_result.is_data():
        raise Exception(f"Server error: {server_result.print(False)}")
    self.alive = True
    self.server = server_result

extras

BaseWidget

Source code in toolboxv2/utils/extras/base_widget.py

class BaseWidget:
    def __init__(self, name: str):
        self.name = name
        self.openWidgetsIDs = {}
        self.onReload = []
        self.iframes = {}

    def register(self, app, fuction, version=None, name="get_widget", level=1, **kwargs):
        if version is None:
            version = app.version
        app.tb(mod_name=self.name, version=version, request_as_kwarg=True, level=level, api=True, name=name, **kwargs)(
            fuction)

    def modify_iterator(self, iterator, replace):
        """
        ['a', 'b'] -> [{replace[0]: 'a',..., replace[len(replace)-1]: 'a'},
        {replace[0]: 'b',..., replace[len(replace)-1]: 'b'}, ]
        """

        for item in iterator:
            modified_item = {replace[i]: (self.name if replace[i] == "name" else '') + item for i in
                             range(len(replace))}
            yield modified_item

    def register2reload(self, *functions):
        for fuction in functions:
            def x(r):
                return fuction(request=r)
            self.onReload.append(x)

    def reload_guard(self, function):
        c = None
        if len(self.onReload) == 0:
            c = function()
        return c

    async def oa_reload_guard(self, function):
        c = None
        if len(self.onReload) == 0:
            c = await function() if asyncio.iscoroutinefunction(function) else function()
        return c

    @staticmethod
    def get_a_group(asset_name, template=None, file_path=None, a_kwargs=None):
        if a_kwargs is None:
            raise ValueError("a_kwargs must be specified")
        return [{'name': asset_name,
                 'file_path': file_path,
                 'kwargs': a_kwargs
                 } if file_path is not None else {'name': asset_name,
                                                  'template': template,
                                                  'kwargs': a_kwargs
                                                  }]

    def group_generator(self, asset_name: str, iterator: iter, template=None, file_path=None, a_kwargs=None):
        groups = []
        work_kwargs = a_kwargs
        for _i, data in enumerate(iterator):
            if isinstance(data, dict):
                work_kwargs = {**a_kwargs, **data}
            groups.append(self.get_a_group(asset_name, template=template, file_path=file_path, a_kwargs=work_kwargs))
        return groups

    def asset_loder(self, app, name, asset_id, file_path=None, template=None, iterator=None, **kwargs):
        a_kwargs = {**{
            'root': f"/api/{self.name}",
            'WidgetID': asset_id},
                    **kwargs}
        asset_name = f"{name}-{asset_id}"
        if iterator is None:
            group = self.get_a_group(asset_name,
                                     template=template,
                                     file_path=file_path,
                                     a_kwargs=a_kwargs)
        else:
            group = self.group_generator(asset_name,
                                         iterator=iterator,
                                         template=template,
                                         file_path=file_path,
                                         a_kwargs=a_kwargs)

        asset = app.run_any(MINIMALHTML.ADD_COLLECTION_TO_GROUP,
                            group_name=self.name,
                            collection={'name': f"{asset_name}",
                                        'group': group},
                            get_results=True)
        if asset.is_error():
            app.run_any(MINIMALHTML.ADD_GROUP, command=self.name)
            asset = app.run_any(MINIMALHTML.ADD_COLLECTION_TO_GROUP,
                                group_name=self.name,
                                collection={'name': f"{self.name}-{asset_name}",
                                            'group': group},
                                get_results=True)
        return asset

    def generate_html(self, app, name="MainWidget", asset_id=str(uuid.uuid4())[:4]):
        return app.run_any(MINIMALHTML.GENERATE_HTML,
                           group_name=self.name,
                           collection_name=f"{name}-{asset_id}")

    def load_widget(self, app, request, name="MainWidget", asset_id=str(uuid.uuid4())[:4]):
        app.run_any(MINIMALHTML.ADD_GROUP, command=self.name)
        self.reload(request)
        html_widget = self.generate_html(app, name, asset_id)
        return html_widget[0]['html_element']

    @staticmethod
    async def get_user_from_request(app, request):
        from toolboxv2.mods.CloudM import User
        if request is None:
            return User()
        return await get_current_user_from_request(app, request)

    @staticmethod
    def get_s_id(request):
        from ..system.types import Result
        if request is None:
            return Result.default_internal_error("No request specified")
        return Result.ok(request.session.get('ID', ''))

    def reload(self, request):
        [_(request) for _ in self.onReload]

    async def oa_reload(self, request):
        [_(request) if not asyncio.iscoroutinefunction(_) else await _(request) for _ in self.onReload]

    async def get_widget(self, request, **kwargs):
        raise NotImplementedError

    def hash_wrapper(self, _id, _salt=''):
        from ..security.cryp import Code
        return Code.one_way_hash(text=_id, salt=_salt, pepper=self.name)

    def register_iframe(self, iframe_id: str, src: str, width: str = "100%", height: str = "500px", **kwargs):
        """
        Registriert einen iframe mit gegebener ID und Quelle

        Args:
            iframe_id: Eindeutige ID für den iframe
            src: URL oder Pfad zur Quelle des iframes
            width: Breite des iframes (default: "100%")
            height: Höhe des iframes (default: "500px")
            **kwargs: Weitere iframe-Attribute
        """
        iframe_config = {
            'src': src,
            'width': width,
            'height': height,
            **kwargs
        }
        self.iframes[iframe_id] = iframe_config

    def create_iframe_asset(self, app, iframe_id: str, asset_id: str = None):
        """
        Erstellt ein Asset für einen registrierten iframe

        Args:
            app: App-Instanz
            iframe_id: ID des registrierten iframes
            asset_id: Optional, spezifische Asset-ID
        """
        if iframe_id not in self.iframes:
            raise ValueError(f"iframe mit ID {iframe_id} nicht registriert")

        if asset_id is None:
            asset_id = str(uuid.uuid4())[:4]

        iframe_config = self.iframes[iframe_id]
        iframe_template = """
        <iframe id="{iframe_id}"
                src="{src}"
                width="{width}"
                height="{height}"
                frameborder="0"
                {additional_attrs}></iframe>
        """.strip()

        # Filtere bekannte Attribute heraus und erstelle String für zusätzliche Attribute
        known_attrs = {'src', 'width', 'height'}
        additional_attrs = ' '.join(
            f'{k}="{v}"' for k, v in iframe_config.items()
            if k not in known_attrs
        )

        iframe_html = iframe_template.format(
            iframe_id=iframe_id,
            src=iframe_config['src'],
            width=iframe_config['width'],
            height=iframe_config['height'],
            additional_attrs=additional_attrs
        )

        return self.asset_loder(
            app=app,
            name=f"iframe-{iframe_id}",
            asset_id=asset_id,
            template=iframe_html
        )

    def load_iframe(self, app, iframe_id: str, asset_id: str = None):
        """
        Lädt einen registrierten iframe und gibt das HTML-Element zurück

        Args:
            app: App-Instanz
            iframe_id: ID des registrierten iframes
            asset_id: Optional, spezifische Asset-ID
        """
        self.create_iframe_asset(app, iframe_id, asset_id)
        return self.generate_html(app, f"iframe-{iframe_id}", asset_id)[0]['html_element']

create_iframe_asset(app, iframe_id, asset_id=None)

Erstellt ein Asset für einen registrierten iframe

Parameters:

Name	Type	Description	Default
app		App-Instanz	required
iframe_id	str	ID des registrierten iframes	required
asset_id	str	Optional, spezifische Asset-ID	None

Source code in toolboxv2/utils/extras/base_widget.py

def create_iframe_asset(self, app, iframe_id: str, asset_id: str = None):
    """
    Erstellt ein Asset für einen registrierten iframe

    Args:
        app: App-Instanz
        iframe_id: ID des registrierten iframes
        asset_id: Optional, spezifische Asset-ID
    """
    if iframe_id not in self.iframes:
        raise ValueError(f"iframe mit ID {iframe_id} nicht registriert")

    if asset_id is None:
        asset_id = str(uuid.uuid4())[:4]

    iframe_config = self.iframes[iframe_id]
    iframe_template = """
    <iframe id="{iframe_id}"
            src="{src}"
            width="{width}"
            height="{height}"
            frameborder="0"
            {additional_attrs}></iframe>
    """.strip()

    # Filtere bekannte Attribute heraus und erstelle String für zusätzliche Attribute
    known_attrs = {'src', 'width', 'height'}
    additional_attrs = ' '.join(
        f'{k}="{v}"' for k, v in iframe_config.items()
        if k not in known_attrs
    )

    iframe_html = iframe_template.format(
        iframe_id=iframe_id,
        src=iframe_config['src'],
        width=iframe_config['width'],
        height=iframe_config['height'],
        additional_attrs=additional_attrs
    )

    return self.asset_loder(
        app=app,
        name=f"iframe-{iframe_id}",
        asset_id=asset_id,
        template=iframe_html
    )

load_iframe(app, iframe_id, asset_id=None)

Lädt einen registrierten iframe und gibt das HTML-Element zurück

Parameters:

Name	Type	Description	Default
app		App-Instanz	required
iframe_id	str	ID des registrierten iframes	required
asset_id	str	Optional, spezifische Asset-ID	None

Source code in toolboxv2/utils/extras/base_widget.py

def load_iframe(self, app, iframe_id: str, asset_id: str = None):
    """
    Lädt einen registrierten iframe und gibt das HTML-Element zurück

    Args:
        app: App-Instanz
        iframe_id: ID des registrierten iframes
        asset_id: Optional, spezifische Asset-ID
    """
    self.create_iframe_asset(app, iframe_id, asset_id)
    return self.generate_html(app, f"iframe-{iframe_id}", asset_id)[0]['html_element']

modify_iterator(iterator, replace)

['a', 'b'] -> [{replace[0]: 'a',..., replace[len(replace)-1]: 'a'}, {replace[0]: 'b',..., replace[len(replace)-1]: 'b'}, ]

Source code in toolboxv2/utils/extras/base_widget.py

def modify_iterator(self, iterator, replace):
    """
    ['a', 'b'] -> [{replace[0]: 'a',..., replace[len(replace)-1]: 'a'},
    {replace[0]: 'b',..., replace[len(replace)-1]: 'b'}, ]
    """

    for item in iterator:
        modified_item = {replace[i]: (self.name if replace[i] == "name" else '') + item for i in
                         range(len(replace))}
        yield modified_item

register_iframe(iframe_id, src, width='100%', height='500px', **kwargs)

Registriert einen iframe mit gegebener ID und Quelle

Parameters:

Name	Type	Description	Default
iframe_id	str	Eindeutige ID für den iframe	required
src	str	URL oder Pfad zur Quelle des iframes	required
width	str	Breite des iframes (default: "100%")	'100%'
height	str	Höhe des iframes (default: "500px")	'500px'
**kwargs		Weitere iframe-Attribute	{}

Source code in toolboxv2/utils/extras/base_widget.py

def register_iframe(self, iframe_id: str, src: str, width: str = "100%", height: str = "500px", **kwargs):
    """
    Registriert einen iframe mit gegebener ID und Quelle

    Args:
        iframe_id: Eindeutige ID für den iframe
        src: URL oder Pfad zur Quelle des iframes
        width: Breite des iframes (default: "100%")
        height: Höhe des iframes (default: "500px")
        **kwargs: Weitere iframe-Attribute
    """
    iframe_config = {
        'src': src,
        'width': width,
        'height': height,
        **kwargs
    }
    self.iframes[iframe_id] = iframe_config

ask_question(title, message, yes_callback=None, no_callback=None, **kwargs)

Ask a yes/no question

Source code in toolboxv2/utils/extras/notification.py

def ask_question(title: str, message: str,
                 yes_callback: Callable = None,
                 no_callback: Callable = None, **kwargs) -> Optional[str]:
    """Ask a yes/no question"""
    notifier = create_notification_system()

    actions = [
        NotificationAction("yes", "Yes", yes_callback, is_default=True),
        NotificationAction("no", "No", no_callback)
    ]

    return notifier.show_notification(
        title, message, NotificationType.QUESTION, actions=actions, **kwargs
    )

quick_error(title, message, **kwargs)

Quick error notification

Source code in toolboxv2/utils/extras/notification.py

def quick_error(title: str, message: str, **kwargs):
    """Quick error notification"""
    notifier = create_notification_system()
    notifier.show_notification(title, message, NotificationType.ERROR, **kwargs)

quick_info(title, message, **kwargs)

Quick info notification

Source code in toolboxv2/utils/extras/notification.py

def quick_info(title: str, message: str, **kwargs):
    """Quick info notification"""
    notifier = create_notification_system()
    notifier.show_notification(title, message, NotificationType.INFO, **kwargs)

quick_success(title, message, **kwargs)

Quick success notification

Source code in toolboxv2/utils/extras/notification.py

def quick_success(title: str, message: str, **kwargs):
    """Quick success notification"""
    notifier = create_notification_system()
    notifier.show_notification(title, message, NotificationType.SUCCESS, **kwargs)

quick_warning(title, message, **kwargs)

Quick warning notification

Source code in toolboxv2/utils/extras/notification.py

def quick_warning(title: str, message: str, **kwargs):
    """Quick warning notification"""
    notifier = create_notification_system()
    notifier.show_notification(title, message, NotificationType.WARNING, **kwargs)

Style

Spinner

Enhanced Spinner with tqdm-like line rendering.

Source code in toolboxv2/utils/extras/Style.py

class Spinner:
    """
    Enhanced Spinner with tqdm-like line rendering.
    """
    SYMBOL_SETS = {
        "c": ["◐", "◓", "◑", "◒"],
        "b": ["▁", "▃", "▄", "▅", "▆", "▇", "█", "▇", "▆", "▅", "▄", "▃"],
        "d": ["⣾", "⣽", "⣻", "⢿", "⡿", "⣟", "⣯", "⣷"],
        "w": ["🌍", "🌎", "🌏"],
        "s": ["🌀   ", " 🌀  ", "  🌀 ", "   🌀", "  🌀 ", " 🌀  "],
        "+": ["+", "x"],
        "t": ["✶", "✸", "✹", "✺", "✹", "✷"]
    }

    def __init__(
        self,
        message: str = "Loading...",
        delay: float = 0.1,
        symbols=None,
        count_down: bool = False,
        time_in_s: float = 0
    ):
        """Initialize spinner with flexible configuration."""
        # Resolve symbol set.
        if isinstance(symbols, str):
            symbols = self.SYMBOL_SETS.get(symbols, None)

        # Default symbols if not provided.
        if symbols is None:
            symbols = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]

        # Test mode symbol set.
        if 'unittest' in sys.argv[0]:
            symbols = ['#', '=', '-']

        self.spinner = itertools.cycle(symbols)
        self.delay = delay
        self.message = message
        self.running = False
        self.spinner_thread = None
        self.max_t = time_in_s
        self.contd = count_down

        # Rendering management.
        self._is_primary = False
        self._start_time = 0

        # Central manager.
        self.manager = SpinnerManager()

    def _generate_render_line(self):
        """Generate the primary render line."""
        current_time = time.time()
        if self.contd:
            remaining = max(0, self.max_t - (current_time - self._start_time))
            time_display = f"{remaining:.2f}"
        else:
            time_display = f"{current_time - self._start_time:.2f}"

        symbol = next(self.spinner)
        return f"{symbol} {self.message} | {time_display}"

    def _generate_secondary_info(self):
        """Generate secondary spinner info for additional spinners."""
        return f"{self.message}"

    def __enter__(self):
        """Start the spinner."""
        self.running = True
        self._start_time = time.time()
        self.manager.register_spinner(self)
        return self

    def __exit__(self, exc_type, exc_value, exc_traceback):
        """Stop the spinner."""
        self.running = False
        self.manager.unregister_spinner(self)
        # Clear the spinner's line if it was the primary spinner.
        if self._is_primary:
            sys.stdout.write("\r\033[K")
            sys.stdout.flush()

__enter__()

Start the spinner.

Source code in toolboxv2/utils/extras/Style.py

def __enter__(self):
    """Start the spinner."""
    self.running = True
    self._start_time = time.time()
    self.manager.register_spinner(self)
    return self

__exit__(exc_type, exc_value, exc_traceback)

Stop the spinner.

Source code in toolboxv2/utils/extras/Style.py

def __exit__(self, exc_type, exc_value, exc_traceback):
    """Stop the spinner."""
    self.running = False
    self.manager.unregister_spinner(self)
    # Clear the spinner's line if it was the primary spinner.
    if self._is_primary:
        sys.stdout.write("\r\033[K")
        sys.stdout.flush()

__init__(message='Loading...', delay=0.1, symbols=None, count_down=False, time_in_s=0)

Initialize spinner with flexible configuration.

Source code in toolboxv2/utils/extras/Style.py

def __init__(
    self,
    message: str = "Loading...",
    delay: float = 0.1,
    symbols=None,
    count_down: bool = False,
    time_in_s: float = 0
):
    """Initialize spinner with flexible configuration."""
    # Resolve symbol set.
    if isinstance(symbols, str):
        symbols = self.SYMBOL_SETS.get(symbols, None)

    # Default symbols if not provided.
    if symbols is None:
        symbols = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]

    # Test mode symbol set.
    if 'unittest' in sys.argv[0]:
        symbols = ['#', '=', '-']

    self.spinner = itertools.cycle(symbols)
    self.delay = delay
    self.message = message
    self.running = False
    self.spinner_thread = None
    self.max_t = time_in_s
    self.contd = count_down

    # Rendering management.
    self._is_primary = False
    self._start_time = 0

    # Central manager.
    self.manager = SpinnerManager()

SpinnerManager

Manages multiple spinners to ensure tqdm-like line rendering. Automatically captures SIGINT (Ctrl+C) to stop all spinners.

Source code in toolboxv2/utils/extras/Style.py

class SpinnerManager(metaclass=Singleton):
    """
    Manages multiple spinners to ensure tqdm-like line rendering.
    Automatically captures SIGINT (Ctrl+C) to stop all spinners.
    """
    _instance = None

    def __new__(cls):
        if not cls._instance:
            cls._instance = super().__new__(cls)
            cls._instance._init_manager()
        return cls._instance

    def _init_manager(self):
        """Initialize spinner management resources and register SIGINT handler."""
        self._spinners = []
        self._lock = threading.Lock()
        self._render_thread = None
        self._should_run = False
        try:
            signal.signal(signal.SIGINT, self._signal_handler)
        except ValueError:
            print("Spinner Manager not in the min Thread no signal possible")
            pass

    def _signal_handler(self, signum, frame):
        """Handle SIGINT by stopping all spinners gracefully."""
        with self._lock:
            for spinner in self._spinners:
                spinner.running = False
            self._spinners.clear()
        self._should_run = False
        sys.stdout.write("\r\033[K")  # Clear the spinner's line.
        sys.stdout.flush()
        sys.exit(0)

    def register_spinner(self, spinner):
        """Register a new spinner."""
        with self._lock:
            # First spinner defines the rendering line.
            if not self._spinners:
                spinner._is_primary = True
            self._spinners.append(spinner)
            # Start rendering if not already running.
            if not self._should_run:
                self._should_run = True
                self._render_thread = threading.Thread(
                    target=self._render_loop,
                    daemon=True
                )
                self._render_thread.start()

    def unregister_spinner(self, spinner):
        """Unregister a completed spinner."""
        with self._lock:
            if spinner in self._spinners:
                self._spinners.remove(spinner)

    def _render_loop(self):
        """Continuous rendering loop for all active spinners."""
        while self._should_run:
            if not self._spinners:
                self._should_run = False
                break

            with self._lock:
                # Find primary spinner (first registered).
                primary_spinner = next((s for s in self._spinners if s._is_primary), None)

                if primary_spinner and primary_spinner.running:
                    # Render in the same line.
                    render_line = primary_spinner._generate_render_line()

                    # Append additional spinner info if multiple exist.
                    if len(self._spinners) > 1:
                        secondary_info = " | ".join(
                            s._generate_secondary_info()
                            for s in self._spinners
                            if s is not primary_spinner and s.running
                        )
                        render_line += f" [{secondary_info}]"

                    # Clear line and write.
                    try:
                        sys.stdout.write("\r" + render_line + "\033[K")
                        sys.stdout.flush()
                    except Exception:
                        self._should_run = False

            time.sleep(0.1)  # Render interval.

register_spinner(spinner)

Register a new spinner.

Source code in toolboxv2/utils/extras/Style.py

def register_spinner(self, spinner):
    """Register a new spinner."""
    with self._lock:
        # First spinner defines the rendering line.
        if not self._spinners:
            spinner._is_primary = True
        self._spinners.append(spinner)
        # Start rendering if not already running.
        if not self._should_run:
            self._should_run = True
            self._render_thread = threading.Thread(
                target=self._render_loop,
                daemon=True
            )
            self._render_thread.start()

unregister_spinner(spinner)

Unregister a completed spinner.

Source code in toolboxv2/utils/extras/Style.py

def unregister_spinner(self, spinner):
    """Unregister a completed spinner."""
    with self._lock:
        if spinner in self._spinners:
            self._spinners.remove(spinner)

base_widget

BaseWidget

Source code in toolboxv2/utils/extras/base_widget.py

class BaseWidget:
    def __init__(self, name: str):
        self.name = name
        self.openWidgetsIDs = {}
        self.onReload = []
        self.iframes = {}

    def register(self, app, fuction, version=None, name="get_widget", level=1, **kwargs):
        if version is None:
            version = app.version
        app.tb(mod_name=self.name, version=version, request_as_kwarg=True, level=level, api=True, name=name, **kwargs)(
            fuction)

    def modify_iterator(self, iterator, replace):
        """
        ['a', 'b'] -> [{replace[0]: 'a',..., replace[len(replace)-1]: 'a'},
        {replace[0]: 'b',..., replace[len(replace)-1]: 'b'}, ]
        """

        for item in iterator:
            modified_item = {replace[i]: (self.name if replace[i] == "name" else '') + item for i in
                             range(len(replace))}
            yield modified_item

    def register2reload(self, *functions):
        for fuction in functions:
            def x(r):
                return fuction(request=r)
            self.onReload.append(x)

    def reload_guard(self, function):
        c = None
        if len(self.onReload) == 0:
            c = function()
        return c

    async def oa_reload_guard(self, function):
        c = None
        if len(self.onReload) == 0:
            c = await function() if asyncio.iscoroutinefunction(function) else function()
        return c

    @staticmethod
    def get_a_group(asset_name, template=None, file_path=None, a_kwargs=None):
        if a_kwargs is None:
            raise ValueError("a_kwargs must be specified")
        return [{'name': asset_name,
                 'file_path': file_path,
                 'kwargs': a_kwargs
                 } if file_path is not None else {'name': asset_name,
                                                  'template': template,
                                                  'kwargs': a_kwargs
                                                  }]

    def group_generator(self, asset_name: str, iterator: iter, template=None, file_path=None, a_kwargs=None):
        groups = []
        work_kwargs = a_kwargs
        for _i, data in enumerate(iterator):
            if isinstance(data, dict):
                work_kwargs = {**a_kwargs, **data}
            groups.append(self.get_a_group(asset_name, template=template, file_path=file_path, a_kwargs=work_kwargs))
        return groups

    def asset_loder(self, app, name, asset_id, file_path=None, template=None, iterator=None, **kwargs):
        a_kwargs = {**{
            'root': f"/api/{self.name}",
            'WidgetID': asset_id},
                    **kwargs}
        asset_name = f"{name}-{asset_id}"
        if iterator is None:
            group = self.get_a_group(asset_name,
                                     template=template,
                                     file_path=file_path,
                                     a_kwargs=a_kwargs)
        else:
            group = self.group_generator(asset_name,
                                         iterator=iterator,
                                         template=template,
                                         file_path=file_path,
                                         a_kwargs=a_kwargs)

        asset = app.run_any(MINIMALHTML.ADD_COLLECTION_TO_GROUP,
                            group_name=self.name,
                            collection={'name': f"{asset_name}",
                                        'group': group},
                            get_results=True)
        if asset.is_error():
            app.run_any(MINIMALHTML.ADD_GROUP, command=self.name)
            asset = app.run_any(MINIMALHTML.ADD_COLLECTION_TO_GROUP,
                                group_name=self.name,
                                collection={'name': f"{self.name}-{asset_name}",
                                            'group': group},
                                get_results=True)
        return asset

    def generate_html(self, app, name="MainWidget", asset_id=str(uuid.uuid4())[:4]):
        return app.run_any(MINIMALHTML.GENERATE_HTML,
                           group_name=self.name,
                           collection_name=f"{name}-{asset_id}")

    def load_widget(self, app, request, name="MainWidget", asset_id=str(uuid.uuid4())[:4]):
        app.run_any(MINIMALHTML.ADD_GROUP, command=self.name)
        self.reload(request)
        html_widget = self.generate_html(app, name, asset_id)
        return html_widget[0]['html_element']

    @staticmethod
    async def get_user_from_request(app, request):
        from toolboxv2.mods.CloudM import User
        if request is None:
            return User()
        return await get_current_user_from_request(app, request)

    @staticmethod
    def get_s_id(request):
        from ..system.types import Result
        if request is None:
            return Result.default_internal_error("No request specified")
        return Result.ok(request.session.get('ID', ''))

    def reload(self, request):
        [_(request) for _ in self.onReload]

    async def oa_reload(self, request):
        [_(request) if not asyncio.iscoroutinefunction(_) else await _(request) for _ in self.onReload]

    async def get_widget(self, request, **kwargs):
        raise NotImplementedError

    def hash_wrapper(self, _id, _salt=''):
        from ..security.cryp import Code
        return Code.one_way_hash(text=_id, salt=_salt, pepper=self.name)

    def register_iframe(self, iframe_id: str, src: str, width: str = "100%", height: str = "500px", **kwargs):
        """
        Registriert einen iframe mit gegebener ID und Quelle

        Args:
            iframe_id: Eindeutige ID für den iframe
            src: URL oder Pfad zur Quelle des iframes
            width: Breite des iframes (default: "100%")
            height: Höhe des iframes (default: "500px")
            **kwargs: Weitere iframe-Attribute
        """
        iframe_config = {
            'src': src,
            'width': width,
            'height': height,
            **kwargs
        }
        self.iframes[iframe_id] = iframe_config

    def create_iframe_asset(self, app, iframe_id: str, asset_id: str = None):
        """
        Erstellt ein Asset für einen registrierten iframe

        Args:
            app: App-Instanz
            iframe_id: ID des registrierten iframes
            asset_id: Optional, spezifische Asset-ID
        """
        if iframe_id not in self.iframes:
            raise ValueError(f"iframe mit ID {iframe_id} nicht registriert")

        if asset_id is None:
            asset_id = str(uuid.uuid4())[:4]

        iframe_config = self.iframes[iframe_id]
        iframe_template = """
        <iframe id="{iframe_id}"
                src="{src}"
                width="{width}"
                height="{height}"
                frameborder="0"
                {additional_attrs}></iframe>
        """.strip()

        # Filtere bekannte Attribute heraus und erstelle String für zusätzliche Attribute
        known_attrs = {'src', 'width', 'height'}
        additional_attrs = ' '.join(
            f'{k}="{v}"' for k, v in iframe_config.items()
            if k not in known_attrs
        )

        iframe_html = iframe_template.format(
            iframe_id=iframe_id,
            src=iframe_config['src'],
            width=iframe_config['width'],
            height=iframe_config['height'],
            additional_attrs=additional_attrs
        )

        return self.asset_loder(
            app=app,
            name=f"iframe-{iframe_id}",
            asset_id=asset_id,
            template=iframe_html
        )

    def load_iframe(self, app, iframe_id: str, asset_id: str = None):
        """
        Lädt einen registrierten iframe und gibt das HTML-Element zurück

        Args:
            app: App-Instanz
            iframe_id: ID des registrierten iframes
            asset_id: Optional, spezifische Asset-ID
        """
        self.create_iframe_asset(app, iframe_id, asset_id)
        return self.generate_html(app, f"iframe-{iframe_id}", asset_id)[0]['html_element']

create_iframe_asset(app, iframe_id, asset_id=None)

Erstellt ein Asset für einen registrierten iframe

Parameters:

Name	Type	Description	Default
app		App-Instanz	required
iframe_id	str	ID des registrierten iframes	required
asset_id	str	Optional, spezifische Asset-ID	None

Source code in toolboxv2/utils/extras/base_widget.py

def create_iframe_asset(self, app, iframe_id: str, asset_id: str = None):
    """
    Erstellt ein Asset für einen registrierten iframe

    Args:
        app: App-Instanz
        iframe_id: ID des registrierten iframes
        asset_id: Optional, spezifische Asset-ID
    """
    if iframe_id not in self.iframes:
        raise ValueError(f"iframe mit ID {iframe_id} nicht registriert")

    if asset_id is None:
        asset_id = str(uuid.uuid4())[:4]

    iframe_config = self.iframes[iframe_id]
    iframe_template = """
    <iframe id="{iframe_id}"
            src="{src}"
            width="{width}"
            height="{height}"
            frameborder="0"
            {additional_attrs}></iframe>
    """.strip()

    # Filtere bekannte Attribute heraus und erstelle String für zusätzliche Attribute
    known_attrs = {'src', 'width', 'height'}
    additional_attrs = ' '.join(
        f'{k}="{v}"' for k, v in iframe_config.items()
        if k not in known_attrs
    )

    iframe_html = iframe_template.format(
        iframe_id=iframe_id,
        src=iframe_config['src'],
        width=iframe_config['width'],
        height=iframe_config['height'],
        additional_attrs=additional_attrs
    )

    return self.asset_loder(
        app=app,
        name=f"iframe-{iframe_id}",
        asset_id=asset_id,
        template=iframe_html
    )

load_iframe(app, iframe_id, asset_id=None)

Lädt einen registrierten iframe und gibt das HTML-Element zurück

Parameters:

Name	Type	Description	Default
app		App-Instanz	required
iframe_id	str	ID des registrierten iframes	required
asset_id	str	Optional, spezifische Asset-ID	None

Source code in toolboxv2/utils/extras/base_widget.py

def load_iframe(self, app, iframe_id: str, asset_id: str = None):
    """
    Lädt einen registrierten iframe und gibt das HTML-Element zurück

    Args:
        app: App-Instanz
        iframe_id: ID des registrierten iframes
        asset_id: Optional, spezifische Asset-ID
    """
    self.create_iframe_asset(app, iframe_id, asset_id)
    return self.generate_html(app, f"iframe-{iframe_id}", asset_id)[0]['html_element']

modify_iterator(iterator, replace)

['a', 'b'] -> [{replace[0]: 'a',..., replace[len(replace)-1]: 'a'}, {replace[0]: 'b',..., replace[len(replace)-1]: 'b'}, ]

Source code in toolboxv2/utils/extras/base_widget.py

def modify_iterator(self, iterator, replace):
    """
    ['a', 'b'] -> [{replace[0]: 'a',..., replace[len(replace)-1]: 'a'},
    {replace[0]: 'b',..., replace[len(replace)-1]: 'b'}, ]
    """

    for item in iterator:
        modified_item = {replace[i]: (self.name if replace[i] == "name" else '') + item for i in
                         range(len(replace))}
        yield modified_item

register_iframe(iframe_id, src, width='100%', height='500px', **kwargs)

Registriert einen iframe mit gegebener ID und Quelle

Parameters:

Name	Type	Description	Default
iframe_id	str	Eindeutige ID für den iframe	required
src	str	URL oder Pfad zur Quelle des iframes	required
width	str	Breite des iframes (default: "100%")	'100%'
height	str	Höhe des iframes (default: "500px")	'500px'
**kwargs		Weitere iframe-Attribute	{}

Source code in toolboxv2/utils/extras/base_widget.py

def register_iframe(self, iframe_id: str, src: str, width: str = "100%", height: str = "500px", **kwargs):
    """
    Registriert einen iframe mit gegebener ID und Quelle

    Args:
        iframe_id: Eindeutige ID für den iframe
        src: URL oder Pfad zur Quelle des iframes
        width: Breite des iframes (default: "100%")
        height: Höhe des iframes (default: "500px")
        **kwargs: Weitere iframe-Attribute
    """
    iframe_config = {
        'src': src,
        'width': width,
        'height': height,
        **kwargs
    }
    self.iframes[iframe_id] = iframe_config

blobs

BlobFile

Source code in toolboxv2/utils/extras/blobs.py

class BlobFile(io.IOBase):
    def __init__(self, filename: str, mode: str = 'r', storage: BlobStorage = None, key: str = None,
                 servers: list[str] = None):
        if not isinstance(filename, str) or not filename:
            raise ValueError("Filename must be a non-empty string.")
        if not filename.startswith('/'): filename = '/' + filename
        self.filename = filename.lstrip('/\\')
        self.blob_id, self.folder, self.datei = self._path_splitter(self.filename)
        self.mode = mode

        if storage is None:
            # In a real app, dependency injection or a global factory would be better
            # but this provides a fallback for simple scripts.
            if not servers:
                from toolboxv2 import get_app
                storage = get_app(from_="BlobStorage").root_blob_storage
            else:
                storage = BlobStorage(servers=servers)

        self.storage = storage
        self.data_buffer = b""
        self.key = key
        if key:
            try:
                assert Code.decrypt_symmetric(Code.encrypt_symmetric(b"test", key), key, to_str=False) == b"test"
            except Exception:
                raise ValueError("Invalid symmetric key provided.")

    @staticmethod
    def _path_splitter(filename):
        parts = Path(filename).parts
        if not parts: raise ValueError("Filename cannot be empty.")
        blob_id = parts[0]
        if len(parts) == 1: raise ValueError("Filename must include a path within the blob, e.g., 'blob_id/file.txt'")
        datei = parts[-1]
        folder = '|'.join(parts[1:-1])
        return blob_id, folder, datei

    def create(self):
        self.storage.create_blob(pickle.dumps({}), self.blob_id)
        return self

    def __enter__(self):
        try:
            raw_blob_data = self.storage.read_blob(self.blob_id)
            if raw_blob_data != b'' and (not raw_blob_data or raw_blob_data is None):
                raw_blob_data = b""
            blob_content = pickle.loads(raw_blob_data)
        except (requests.exceptions.HTTPError, EOFError, pickle.UnpicklingError, ConnectionError) as e:
            if isinstance(e, requests.exceptions.HTTPError) and e.response.status_code == 404:
                blob_content = {}  # Blob doesn't exist yet, treat as empty
            elif isinstance(e, EOFError | pickle.UnpicklingError):
                blob_content = {}  # Blob is empty or corrupt, treat as empty for writing
            else:
                self.storage.create_blob(blob_id=self.blob_id, data=pickle.dumps({}))
                blob_content = {}

        if 'r' in self.mode:
            path_key = self.folder if self.folder else self.datei
            if self.folder:
                file_data = blob_content.get(self.folder, {}).get(self.datei)
            else:
                file_data = blob_content.get(self.datei)

            if file_data:
                self.data_buffer = file_data
                if self.key:
                    self.data_buffer = Code.decrypt_symmetric(self.data_buffer, self.key, to_str=False)
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        if 'w' in self.mode:
            final_data = self.data_buffer
            if self.key:
                final_data = Code.encrypt_symmetric(final_data, self.key)

            try:
                raw_blob_data = self.storage.read_blob(self.blob_id)
                blob_content = pickle.loads(raw_blob_data)
            except Exception:
                blob_content = {}

            # Safely navigate and create path
            current_level = blob_content
            if self.folder:
                if self.folder not in current_level:
                    current_level[self.folder] = {}
                current_level = current_level[self.folder]

            current_level[self.datei] = final_data
            self.storage.update_blob(self.blob_id, pickle.dumps(blob_content))




    def exists(self) -> bool:
        """
        Checks if the specific file path exists within the blob without reading its content.
        This is an efficient, read-only operation.

        Returns:
            bool: True if the file exists within the blob, False otherwise.
        """
        try:
            # Fetch the raw blob data. This leverages the local cache if available.
            raw_blob_data = self.storage.read_blob(self.blob_id)
            # Unpickle the directory structure.
            if raw_blob_data:
                blob_content = pickle.loads(raw_blob_data)
            else:
                return False
        except (requests.exceptions.HTTPError, EOFError, pickle.UnpicklingError, ConnectionError):
            # If the blob itself doesn't exist, is empty, or can't be reached,
            # then the file within it cannot exist.
            return False

        # Navigate the dictionary to check for the file's existence.
        current_level = blob_content
        if self.folder:
            if self.folder not in current_level:
                return False
            current_level = current_level[self.folder]

        return self.datei in current_level

    def clear(self):
        self.data_buffer = b''

    def write(self, data):
        if 'w' not in self.mode: raise OSError("File not opened in write mode.")
        if isinstance(data, str):
            self.data_buffer += data.encode()
        elif isinstance(data, bytes):
            self.data_buffer += data
        else:
            raise TypeError("write() argument must be str or bytes")

    def read(self):
        if 'r' not in self.mode: raise OSError("File not opened in read mode.")
        return self.data_buffer

    def read_json(self):
        if 'r' not in self.mode: raise ValueError("File not opened in read mode.")
        if self.data_buffer == b"": return {}
        return json.loads(self.data_buffer.decode())

    def write_json(self, data):
        if 'w' not in self.mode: raise ValueError("File not opened in write mode.")
        self.data_buffer += json.dumps(data).encode()

    def read_pickle(self):
        if 'r' not in self.mode: raise ValueError("File not opened in read mode.")
        if self.data_buffer == b"": return {}
        return pickle.loads(self.data_buffer)

    def write_pickle(self, data):
        if 'w' not in self.mode: raise ValueError("File not opened in write mode.")
        self.data_buffer += pickle.dumps(data)

    def read_yaml(self):
        if 'r' not in self.mode: raise ValueError("File not opened in read mode.")
        if self.data_buffer == b"": return {}
        return yaml.safe_load(self.data_buffer)

    def write_yaml(self, data):
        if 'w' not in self.mode: raise ValueError("File not opened in write mode.")
        yaml.dump(data, self)

exists()

Checks if the specific file path exists within the blob without reading its content. This is an efficient, read-only operation.

Returns:

Name	Type	Description
bool	bool	True if the file exists within the blob, False otherwise.

Source code in toolboxv2/utils/extras/blobs.py

def exists(self) -> bool:
    """
    Checks if the specific file path exists within the blob without reading its content.
    This is an efficient, read-only operation.

    Returns:
        bool: True if the file exists within the blob, False otherwise.
    """
    try:
        # Fetch the raw blob data. This leverages the local cache if available.
        raw_blob_data = self.storage.read_blob(self.blob_id)
        # Unpickle the directory structure.
        if raw_blob_data:
            blob_content = pickle.loads(raw_blob_data)
        else:
            return False
    except (requests.exceptions.HTTPError, EOFError, pickle.UnpicklingError, ConnectionError):
        # If the blob itself doesn't exist, is empty, or can't be reached,
        # then the file within it cannot exist.
        return False

    # Navigate the dictionary to check for the file's existence.
    current_level = blob_content
    if self.folder:
        if self.folder not in current_level:
            return False
        current_level = current_level[self.folder]

    return self.datei in current_level

BlobStorage

A production-ready client for the distributed blob storage server. It handles communication with a list of server instances, manages a local cache, and implements backoff/retry logic for resilience.

Source code in toolboxv2/utils/extras/blobs.py

class BlobStorage:
    """
    A production-ready client for the distributed blob storage server.
    It handles communication with a list of server instances, manages a local cache,
    and implements backoff/retry logic for resilience.
    """

    def __init__(self, servers: list[str], storage_directory: str = './.data/blob_cache'):


        self.servers = servers
        self.session = requests.Session()
        self.storage_directory = storage_directory
        self.blob_ids = []
        os.makedirs(storage_directory, exist_ok=True)

        # Initialize the consistent hash ring
        self.hash_ring = ConsistentHashRing()
        for server in self.servers:
            self.hash_ring.add_node(server)

    def _make_request(self, method, endpoint, blob_id: str = None, max_retries=2, **kwargs):
        """
        Makes a resilient HTTP request to the server cluster.
        - If a blob_id is provided, it uses the consistent hash ring to find the
          primary server and subsequent backup servers in a predictable order.
        - If no blob_id is given (e.g., for broadcast actions), it tries servers randomly.
        - Implements exponential backoff on server errors.
        """
        if not self.servers:
            res = requests.Response()
            res.status_code = 503
            res.reason = "No servers available"
            return res

        if blob_id:
            # Get the ordered list of servers for this specific blob
            preferred_servers = self.hash_ring.get_nodes_for_key(blob_id)
        else:
            # For non-specific requests, shuffle all servers
            preferred_servers = random.sample(self.servers, len(self.servers))

        last_error = None
        for attempt in range(max_retries):
            for server in preferred_servers:
                url = f"{server.rstrip('/')}{endpoint}"
                try:
                    # In a targeted request, print which server we are trying
                    response = self.session.request(method, url, timeout=10, **kwargs)

                    if 500 <= response.status_code < 600:
                        get_logger().warning(f"Warning: Server {server} returned status {response.status_code}. Retrying...")
                        continue
                    response.raise_for_status()
                    return response
                except requests.exceptions.RequestException as e:
                    last_error = e
                    get_logger().warning(f"Warning: Could not connect to server {server}: {e}. Trying next server.")

            if attempt < max_retries - 1:
                wait_time = 2 ** (attempt*0.1)
                get_logger().warning(f"Warning: All preferred servers failed. Retrying in {wait_time} seconds...")
                time.sleep(wait_time)
                if len(preferred_servers) == 1 and len(self.servers) > 1:
                    preferred_servers = random.sample(self.servers, len(self.servers))

        raise ConnectionError(f"Failed to execute request after {max_retries} attempts. Last error: {last_error}")


    def create_blob(self, data: bytes, blob_id=None) -> str:
        """
        Creates a new blob. The blob_id is calculated client-side by hashing
        the content, and the data is sent to the correct server determined
        by the consistent hash ring. This uses a PUT request, making creation
        idempotent.
        """
        # The blob ID is the hash of its content, ensuring content-addressable storage.
        if not blob_id:
            blob_id = hashlib.sha256(data).hexdigest()

        # Use PUT, as we now know the blob's final ID/URL.
        # Pass blob_id to _make_request so it uses the hash ring.
        print(f"Creating blob {blob_id} on {self._make_request('PUT', f'/blob/{blob_id}',blob_id=blob_id, data=data).status_code}")
        # blob_id = response.text
        self._save_blob_to_cache(blob_id, data)
        return blob_id

    def read_blob(self, blob_id: str) -> bytes:
        cached_data = self._load_blob_from_cache(blob_id)
        if cached_data is not None:
            return cached_data

        get_logger().info(f"Info: Blob '{blob_id}' not in cache, fetching from network.")
        # Pass blob_id to _make_request to target the correct server(s).
        response = self._make_request('GET', f'/blob/{blob_id}', blob_id=blob_id)

        blob_data = response.content
        self._save_blob_to_cache(blob_id, blob_data)
        return blob_data

    def update_blob(self, blob_id: str, data: bytes):
        # Pass blob_id to _make_request to target the correct server(s).
        response = self._make_request('PUT', f'/blob/{blob_id}', blob_id=blob_id, data=data)
        self._save_blob_to_cache(blob_id, data)
        return response

    def delete_blob(self, blob_id: str):
        # Pass blob_id to _make_request to target the correct server(s).
        self._make_request('DELETE', f'/blob/{blob_id}', blob_id=blob_id)
        cache_file = self._get_blob_cache_filename(blob_id)
        if os.path.exists(cache_file):
            os.remove(cache_file)

    # NOTE: share_blobs and recover_blob are coordination endpoints. They do not
    # act on a single blob, so they will continue to use the non-targeted (random)
    # request mode to contact any available server to act as a coordinator.
    def share_blobs(self, blob_ids: list[str]):
        get_logger().info(f"Info: Instructing a server to share blobs for recovery: {blob_ids}")
        payload = {"blob_ids": blob_ids}
        # No blob_id passed, will try any server as a coordinator.
        self._make_request('POST', '/share', json=payload)
        get_logger().info("Info: Sharing command sent successfully.")

    def recover_blob(self, lost_blob_id: str) -> bytes:
        get_logger().info(f"Info: Attempting to recover '{lost_blob_id}' from the cluster.")
        payload = {"blob_id": lost_blob_id}
        # No blob_id passed, recovery can be initiated by any server.
        response = self._make_request('POST', '/recover', json=payload)

        recovered_data = response.content
        get_logger().info(f"Info: Successfully recovered blob '{lost_blob_id}'.")
        self._save_blob_to_cache(lost_blob_id, recovered_data)
        return recovered_data

    def _get_blob_cache_filename(self, blob_id: str) -> str:
        return os.path.join(self.storage_directory, blob_id + '.blobcache')

    def _save_blob_to_cache(self, blob_id: str, data: bytes):
        if not data or data is None:
            return
        if blob_id not in self.blob_ids:
            self.blob_ids.append(blob_id)
        with open(self._get_blob_cache_filename(blob_id), 'wb') as f:
            f.write(data)

    def _load_blob_from_cache(self, blob_id: str) -> bytes | None:
        cache_file = self._get_blob_cache_filename(blob_id)
        if not os.path.exists(cache_file):
            return None
        with open(cache_file, 'rb') as f:
            return f.read()

    def exit(self):
        if len(self.blob_ids) < 5:
            return
        for _i in range(len(self.servers)//2+1):
            self.share_blobs(self.blob_ids)

create_blob(data, blob_id=None)

Creates a new blob. The blob_id is calculated client-side by hashing the content, and the data is sent to the correct server determined by the consistent hash ring. This uses a PUT request, making creation idempotent.

Source code in toolboxv2/utils/extras/blobs.py

def create_blob(self, data: bytes, blob_id=None) -> str:
    """
    Creates a new blob. The blob_id is calculated client-side by hashing
    the content, and the data is sent to the correct server determined
    by the consistent hash ring. This uses a PUT request, making creation
    idempotent.
    """
    # The blob ID is the hash of its content, ensuring content-addressable storage.
    if not blob_id:
        blob_id = hashlib.sha256(data).hexdigest()

    # Use PUT, as we now know the blob's final ID/URL.
    # Pass blob_id to _make_request so it uses the hash ring.
    print(f"Creating blob {blob_id} on {self._make_request('PUT', f'/blob/{blob_id}',blob_id=blob_id, data=data).status_code}")
    # blob_id = response.text
    self._save_blob_to_cache(blob_id, data)
    return blob_id

ConsistentHashRing

A consistent hash ring implementation to map keys (blob_ids) to nodes (servers). It uses virtual nodes (replicas) to ensure a more uniform distribution of keys.

Source code in toolboxv2/utils/extras/blobs.py

class ConsistentHashRing:
    """
    A consistent hash ring implementation to map keys (blob_ids) to nodes (servers).
    It uses virtual nodes (replicas) to ensure a more uniform distribution of keys.
    """
    def __init__(self, replicas=100):
        """
        :param replicas: The number of virtual nodes for each physical node.
                         Higher values lead to more balanced distribution.
        """
        self.replicas = replicas
        self._keys = []  # Sorted list of hash values (the ring)
        self._nodes = {} # Maps hash values to physical node URLs

    def _hash(self, key: str) -> int:
        """Hashes a key to an integer using md5 for speed and distribution."""
        return int(hashlib.md5(key.encode('utf-8')).hexdigest(), 16)

    def add_node(self, node: str):
        """Adds a physical node to the hash ring."""
        for i in range(self.replicas):
            vnode_key = f"{node}:{i}"
            h = self._hash(vnode_key)
            bisect.insort(self._keys, h)
            self._nodes[h] = node

    def get_nodes_for_key(self, key: str) -> list[str]:
        """
        Returns an ordered list of nodes responsible for the given key.
        The first node in the list is the primary, the rest are failover candidates
        in preferential order.
        """
        if not self._nodes:
            return []

        h = self._hash(key)
        start_idx = bisect.bisect_left(self._keys, h)

        # Collect unique physical nodes by iterating around the ring
        found_nodes = []
        for i in range(len(self._keys)):
            idx = (start_idx + i) % len(self._keys)
            node_hash = self._keys[idx]
            physical_node = self._nodes[node_hash]
            if physical_node not in found_nodes:
                found_nodes.append(physical_node)
            # Stop when we have found all unique physical nodes
            if len(found_nodes) == len(set(self._nodes.values())):
                break
        return found_nodes

__init__(replicas=100)

:param replicas: The number of virtual nodes for each physical node. Higher values lead to more balanced distribution.

Source code in toolboxv2/utils/extras/blobs.py

def __init__(self, replicas=100):
    """
    :param replicas: The number of virtual nodes for each physical node.
                     Higher values lead to more balanced distribution.
    """
    self.replicas = replicas
    self._keys = []  # Sorted list of hash values (the ring)
    self._nodes = {} # Maps hash values to physical node URLs

add_node(node)

Adds a physical node to the hash ring.

Source code in toolboxv2/utils/extras/blobs.py

def add_node(self, node: str):
    """Adds a physical node to the hash ring."""
    for i in range(self.replicas):
        vnode_key = f"{node}:{i}"
        h = self._hash(vnode_key)
        bisect.insort(self._keys, h)
        self._nodes[h] = node

get_nodes_for_key(key)

Returns an ordered list of nodes responsible for the given key. The first node in the list is the primary, the rest are failover candidates in preferential order.

Source code in toolboxv2/utils/extras/blobs.py

def get_nodes_for_key(self, key: str) -> list[str]:
    """
    Returns an ordered list of nodes responsible for the given key.
    The first node in the list is the primary, the rest are failover candidates
    in preferential order.
    """
    if not self._nodes:
        return []

    h = self._hash(key)
    start_idx = bisect.bisect_left(self._keys, h)

    # Collect unique physical nodes by iterating around the ring
    found_nodes = []
    for i in range(len(self._keys)):
        idx = (start_idx + i) % len(self._keys)
        node_hash = self._keys[idx]
        physical_node = self._nodes[node_hash]
        if physical_node not in found_nodes:
            found_nodes.append(physical_node)
        # Stop when we have found all unique physical nodes
        if len(found_nodes) == len(set(self._nodes.values())):
            break
    return found_nodes

gist_control

GistLoader

Source code in toolboxv2/utils/extras/gist_control.py

class GistLoader:
    def __init__(self, gist_url):
        self.gist_url = gist_url
        self.module_code = None

    def load_module(self, module_name):
        """Lädt das Modul mit dem gegebenen Namen."""
        if self.module_code is None:
            self.module_code = self._fetch_gist_content()

        # Erstelle ein neues Modul
        module = importlib.util.module_from_spec(self.get_spec(module_name))
        exec(self.module_code, module.__dict__)
        return module

    def get_spec(self, module_name):
        """Gibt die Modul-Specifikation zurück."""
        return ModuleSpec(module_name, self)

    def get_filename(self, module_name):
        return f"<gist:{self.gist_url}>"

    def _fetch_gist_content(self):
        """Lädt den Inhalt des Gists von der GitHub API herunter."""
        gist_id = self.gist_url.split('/')[-1]
        api_url = f"https://api.github.com/gists/{gist_id}"

        response = requests.get(api_url)

        if response.status_code == 200:
            gist_data = response.json()
            first_file = next(iter(gist_data['files'].values()))
            return first_file['content']
        else:
            raise Exception(f"Failed to fetch gist: {response.status_code}")

get_spec(module_name)

Gibt die Modul-Specifikation zurück.

Source code in toolboxv2/utils/extras/gist_control.py

def get_spec(self, module_name):
    """Gibt die Modul-Specifikation zurück."""
    return ModuleSpec(module_name, self)

load_module(module_name)

Lädt das Modul mit dem gegebenen Namen.

Source code in toolboxv2/utils/extras/gist_control.py

def load_module(self, module_name):
    """Lädt das Modul mit dem gegebenen Namen."""
    if self.module_code is None:
        self.module_code = self._fetch_gist_content()

    # Erstelle ein neues Modul
    module = importlib.util.module_from_spec(self.get_spec(module_name))
    exec(self.module_code, module.__dict__)
    return module

helper_test_functions

generate_edge_value(param_type)

Generiert Edge-Case-Werte basierend auf dem Parametertyp.

Source code in toolboxv2/utils/extras/helper_test_functions.py

def generate_edge_value(param_type: Any) -> Any:
    """
    Generiert Edge-Case-Werte basierend auf dem Parametertyp.
    """
    if param_type in [int, float]:
        return -999  # Beispiel für negative Zahlen
    elif param_type == str:
        return "test " * 100  # Lange zufällige Strings
    # Fügen Sie hier weitere Bedingungen für andere Datentypen hinzu
    return None

generate_normal_value(param_type)

Generiert normale Werte basierend auf dem Parametertyp.

Source code in toolboxv2/utils/extras/helper_test_functions.py

def generate_normal_value(param_type: Any) -> Any:
    """
    Generiert normale Werte basierend auf dem Parametertyp.
    """
    from toolboxv2 import RequestData
    if param_type in [int, float]:
        return random.randint(0, 100)  # Zufällige normale Zahlen
    elif param_type == str:
        return "test" # Zufälliges Wort
    elif param_type == RequestData:
        return RequestData.moc()
    # Fügen Sie hier weitere Bedingungen für andere Datentypen hinzu
    return None

keword_matcher

calculate_keyword_score(text, keywords)

Berechnet den Keyword-Score basierend auf der Häufigkeit der Keywords im Text. Case-insensitive und optimiert für Geschwindigkeit.

:param text: Eingabetext als String :param keywords: Set von Keywords :return: Gesamt-Score als Integer

Source code in toolboxv2/utils/extras/keword_matcher.py

def calculate_keyword_score(text: str, keywords: set[str]) -> int:
    """
    Berechnet den Keyword-Score basierend auf der Häufigkeit der Keywords im Text.
    Case-insensitive und optimiert für Geschwindigkeit.

    :param text: Eingabetext als String
    :param keywords: Set von Keywords
    :return: Gesamt-Score als Integer
    """
    # Vorverarbeitung der Keywords
    keyword_pattern = re.compile(
        r'\b(' + '|'.join(re.escape(k.lower()) for k in keywords) + r')\b',
        flags=re.IGNORECASE
    )

    # Erstelle Frequenz-Wörterbuch
    freq_dict = defaultdict(int)

    # Finde alle Übereinstimmungen
    matches = keyword_pattern.findall(text.lower())

    # Zähle die Treffer
    for match in matches:
        freq_dict[match.lower()] += 1

    # Berechne Gesamt-Score
    total_score = sum(freq_dict.values())

    return total_score

calculate_weighted_score(text, keyword_weights)

Berechnet gewichteten Score mit unterschiedlichen Gewichten pro Keyword

:param text: Eingabetext :param keyword_weights: Dictionary mit {Keyword: Gewicht} :return: Gewichteter Gesamt-Score

Source code in toolboxv2/utils/extras/keword_matcher.py

def calculate_weighted_score(text: str, keyword_weights: dict or list) -> float:
    """
    Berechnet gewichteten Score mit unterschiedlichen Gewichten pro Keyword

    :param text: Eingabetext
    :param keyword_weights: Dictionary mit {Keyword: Gewicht}
    :return: Gewichteter Gesamt-Score
    """
    total = 0.0
    text_lower = text.lower()

    if isinstance(keyword_weights, list):
        keyword_weights = {k:v for k, v in keyword_weights}

    for keyword, weight in keyword_weights.items():
        count = len(re.findall(r'\b' + re.escape(keyword.lower()) + r'\b', text_lower))
        total += count * weight

    return round(total, 2)

extract_keywords(text, max_len=-1, min_word_length=3, with_weights=False, remove_stopwords=True, stopwords=True)

Extrahiert Keywords mit optionaler Frequenzgewichtung

:param text: Eingabetext :param max_len: Maximale Anzahl Keywords (-1 = alle) :param min_word_length: Minimale Wortlänge :param with_weights: Gibt Wort+Frequenz zurück wenn True :param remove_stopwords: Filtert deutsche Stopwörter :param german_stopwords: Verwendet deutsche Standard-Stopwörter :return: Keywords oder (Keyword, Häufigkeit) Paare

Source code in toolboxv2/utils/extras/keword_matcher.py

def extract_keywords(
    text: str,
    max_len: int = -1,
    min_word_length: int = 3,
    with_weights: bool = False,
    remove_stopwords: bool = True,
    stopwords: bool = True
) -> list[str] | list[tuple[str, int]]:
    """
    Extrahiert Keywords mit optionaler Frequenzgewichtung

    :param text: Eingabetext
    :param max_len: Maximale Anzahl Keywords (-1 = alle)
    :param min_word_length: Minimale Wortlänge
    :param with_weights: Gibt Wort+Frequenz zurück wenn True
    :param remove_stopwords: Filtert deutsche Stopwörter
    :param german_stopwords: Verwendet deutsche Standard-Stopwörter
    :return: Keywords oder (Keyword, Häufigkeit) Paare
    """

    # Deutsche Basis-Stopwörter
    DEFAULT_STOPWORDS = STOPWORDS if stopwords else set()

    # Text vorverarbeiten
    words = re.findall(r'\b\w+\b', text.lower())

    # Worte filtern
    filtered_words = [
        word for word in words
        if len(word) > min_word_length
           and (not remove_stopwords or word not in DEFAULT_STOPWORDS)
    ]

    # Frequenzanalyse
    word_counts = defaultdict(int)
    for word in filtered_words:
        word_counts[word] += 1

    # Sortierung: Zuerst Häufigkeit, dann alphabetisch
    sorted_words = sorted(
        word_counts.items(),
        key=lambda x: (-x[1], x[0])
    )

    # Längenbegrenzung
    if max_len == -1:
        max_len = None
    result = sorted_words[:max_len]

    return result if with_weights else [word for word, _ in result]

mkdocs

CodeElement dataclass

Represents a code element (class, function, etc.)

Source code in toolboxv2/utils/extras/mkdocs.py

@dataclass
class CodeElement:
    """Represents a code element (class, function, etc.)"""
    name: str
    element_type: str
    file_path: str
    line_start: int
    line_end: int
    signature: str
    docstring: Optional[str] = None
    hash_signature: str = ""
    parent_class: Optional[str] = None

CodeElementExtractor

Extracts code elements from source files

Source code in toolboxv2/utils/extras/mkdocs.py

class CodeElementExtractor:
    """Extracts code elements from source files"""

    def extract_python_elements(self, file_path: Path) -> List[CodeElement]:
        """Extract elements from Python file"""
        elements = []

        try:
            with open(file_path, 'r', encoding='utf-8') as f:
                content = f.read()

            tree = ast.parse(content)

            for node in ast.walk(tree):
                if isinstance(node, ast.ClassDef):
                    elements.append(CodeElement(
                        name=node.name,
                        element_type='class',
                        file_path=str(file_path),
                        line_start=node.lineno,
                        line_end=getattr(node, 'end_lineno', node.lineno),
                        signature=f"class {node.name}",
                        docstring=ast.get_docstring(node),
                        hash_signature=self._hash_node(node, content)
                    ))

                    # Extract methods
                    for item in node.body:
                        if isinstance(item, ast.FunctionDef):
                            elements.append(CodeElement(
                                name=item.name,
                                element_type='method',
                                file_path=str(file_path),
                                line_start=item.lineno,
                                line_end=getattr(item, 'end_lineno', item.lineno),
                                signature=self._get_function_signature(item),
                                docstring=ast.get_docstring(item),
                                parent_class=node.name,
                                hash_signature=self._hash_node(item, content)
                            ))

                elif isinstance(node, ast.FunctionDef):
                    elements.append(CodeElement(
                        name=node.name,
                        element_type='function',
                        file_path=str(file_path),
                        line_start=node.lineno,
                        line_end=getattr(node, 'end_lineno', node.lineno),
                        signature=self._get_function_signature(node),
                        docstring=ast.get_docstring(node),
                        hash_signature=self._hash_node(node, content)
                    ))

        except Exception as e:
            logger.error(f"Error extracting Python elements from {file_path}: {e}")

        return elements

    def _get_function_signature(self, node: ast.FunctionDef) -> str:
        """Get function signature string"""
        args = []
        for arg in node.args.args:
            args.append(arg.arg)

        return f"def {node.name}({', '.join(args)})"

    def _hash_node(self, node: ast.AST, content: str) -> str:
        """Generate hash for AST node"""
        try:
            node_source = ast.unparse(node)
            return hashlib.md5(node_source.encode()).hexdigest()
        except:
            # Fallback: use line-based hash
            lines = content.split('\n')
            start = getattr(node, 'lineno', 1) - 1
            end = getattr(node, 'end_lineno', start + 1)
            node_content = '\n'.join(lines[start:end])
            return hashlib.md5(node_content.encode()).hexdigest()

extract_python_elements(file_path)

Extract elements from Python file

Source code in toolboxv2/utils/extras/mkdocs.py

def extract_python_elements(self, file_path: Path) -> List[CodeElement]:
    """Extract elements from Python file"""
    elements = []

    try:
        with open(file_path, 'r', encoding='utf-8') as f:
            content = f.read()

        tree = ast.parse(content)

        for node in ast.walk(tree):
            if isinstance(node, ast.ClassDef):
                elements.append(CodeElement(
                    name=node.name,
                    element_type='class',
                    file_path=str(file_path),
                    line_start=node.lineno,
                    line_end=getattr(node, 'end_lineno', node.lineno),
                    signature=f"class {node.name}",
                    docstring=ast.get_docstring(node),
                    hash_signature=self._hash_node(node, content)
                ))

                # Extract methods
                for item in node.body:
                    if isinstance(item, ast.FunctionDef):
                        elements.append(CodeElement(
                            name=item.name,
                            element_type='method',
                            file_path=str(file_path),
                            line_start=item.lineno,
                            line_end=getattr(item, 'end_lineno', item.lineno),
                            signature=self._get_function_signature(item),
                            docstring=ast.get_docstring(item),
                            parent_class=node.name,
                            hash_signature=self._hash_node(item, content)
                        ))

            elif isinstance(node, ast.FunctionDef):
                elements.append(CodeElement(
                    name=node.name,
                    element_type='function',
                    file_path=str(file_path),
                    line_start=node.lineno,
                    line_end=getattr(node, 'end_lineno', node.lineno),
                    signature=self._get_function_signature(node),
                    docstring=ast.get_docstring(node),
                    hash_signature=self._hash_node(node, content)
                ))

    except Exception as e:
        logger.error(f"Error extracting Python elements from {file_path}: {e}")

    return elements

DocSection dataclass

Represents a documentation section with change tracking

Source code in toolboxv2/utils/extras/mkdocs.py

@dataclass
class DocSection:
    """Represents a documentation section with change tracking"""
    section_id: str
    file_path: str
    title: str
    content: str
    level: int
    line_start: int
    line_end: int
    source_refs: List[str] = field(default_factory=list)
    tags: List[str] = field(default_factory=list)
    hash_signature: str = ""
    content_hash: str = ""
    last_modified: datetime = field(default_factory=datetime.now)
    change_detected: bool = False

DocsAnalyzer

Analyzes documentation quality and completeness

Source code in toolboxv2/utils/extras/mkdocs.py

class DocsAnalyzer:
    """Analyzes documentation quality and completeness"""

    def __init__(self, index: DocsIndex, project_root: Path):
        self.index = index
        self.project_root = project_root

    def find_unclear_sections(self) -> List[str]:
        """Find sections with unclear or placeholder content"""
        unclear_indicators = [
            "todo", "fixme", "placeholder", "coming soon", "not implemented",
            "tbd", "under construction", "work in progress", "missing",
            "add content here", "fill this", "example here"
        ]

        unclear_sections = []

        for section_id, section in self.index.sections.items():
            content_lower = section.content.lower()

            # Check for unclear indicators
            if any(indicator in content_lower for indicator in unclear_indicators):
                unclear_sections.append(section_id)
                continue

            # Check for very short content (likely incomplete)
            if len(section.content.strip()) < 50:
                unclear_sections.append(section_id)
                continue

            # Check for sections with only code blocks (no explanation)
            code_blocks = re.findall(r'```[\s\S]*?```', section.content)
            text_without_code = re.sub(r'```[\s\S]*?```', '', section.content).strip()
            if code_blocks and len(text_without_code) < 20:
                unclear_sections.append(section_id)

        return unclear_sections

    def find_missing_implementations(self) -> List[Dict]:
        """Find TOC entries that don't have corresponding implementations"""
        missing = []

        for section_id, section in self.index.sections.items():
            # Look for function/class references that don't exist in code
            potential_refs = re.findall(r'`([A-Za-z_][A-Za-z0-9_.]*)`', section.content)

            for ref in potential_refs:
                if '.' in ref:  # Looks like a class.method reference
                    if not any(ref in element_id for element_id in self.index.code_elements.keys()):
                        missing.append({
                            "section_id": section_id,
                            "missing_ref": ref,
                            "type": "code_reference",
                            "content_context": section.content[:200] + "..."
                        })

        return missing

find_missing_implementations()

Find TOC entries that don't have corresponding implementations

Source code in toolboxv2/utils/extras/mkdocs.py

def find_missing_implementations(self) -> List[Dict]:
    """Find TOC entries that don't have corresponding implementations"""
    missing = []

    for section_id, section in self.index.sections.items():
        # Look for function/class references that don't exist in code
        potential_refs = re.findall(r'`([A-Za-z_][A-Za-z0-9_.]*)`', section.content)

        for ref in potential_refs:
            if '.' in ref:  # Looks like a class.method reference
                if not any(ref in element_id for element_id in self.index.code_elements.keys()):
                    missing.append({
                        "section_id": section_id,
                        "missing_ref": ref,
                        "type": "code_reference",
                        "content_context": section.content[:200] + "..."
                    })

    return missing

find_unclear_sections()

Find sections with unclear or placeholder content

Source code in toolboxv2/utils/extras/mkdocs.py

def find_unclear_sections(self) -> List[str]:
    """Find sections with unclear or placeholder content"""
    unclear_indicators = [
        "todo", "fixme", "placeholder", "coming soon", "not implemented",
        "tbd", "under construction", "work in progress", "missing",
        "add content here", "fill this", "example here"
    ]

    unclear_sections = []

    for section_id, section in self.index.sections.items():
        content_lower = section.content.lower()

        # Check for unclear indicators
        if any(indicator in content_lower for indicator in unclear_indicators):
            unclear_sections.append(section_id)
            continue

        # Check for very short content (likely incomplete)
        if len(section.content.strip()) < 50:
            unclear_sections.append(section_id)
            continue

        # Check for sections with only code blocks (no explanation)
        code_blocks = re.findall(r'```[\s\S]*?```', section.content)
        text_without_code = re.sub(r'```[\s\S]*?```', '', section.content).strip()
        if code_blocks and len(text_without_code) < 20:
            unclear_sections.append(section_id)

    return unclear_sections

DocsIndex dataclass

Complete documentation index with section-level tracking

Source code in toolboxv2/utils/extras/mkdocs.py

@dataclass
class DocsIndex:
    """Complete documentation index with section-level tracking"""
    sections: Dict[str, DocSection] = field(default_factory=dict)
    code_elements: Dict[str, CodeElement] = field(default_factory=dict)
    file_hashes: Dict[str, str] = field(default_factory=dict)
    section_hashes: Dict[str, str] = field(default_factory=dict)
    last_git_commit: Optional[str] = None
    last_indexed: datetime = field(default_factory=datetime.now)
    version: str = "1.1"

DocsIndexer

Main indexer that builds and maintains the complete documentation index

Source code in toolboxv2/utils/extras/mkdocs.py

class DocsIndexer:
    """Main indexer that builds and maintains the complete documentation index"""

    def __init__(self, project_root: Path, docs_root: Path,
                 include_dirs: List[str] = None, exclude_dirs: List[str] = None):
        self.project_root = project_root
        self.docs_root = docs_root
        self.git_detector = GitChangeDetector(project_root)
        self.import_analyzer = ImportAnalyzer(project_root)
        self.code_extractor = CodeElementExtractor()
        self.md_parser = MarkdownParser()
        self.index_file = docs_root / '.docs_index.json'

        # Directory filters
        self.include_dirs = include_dirs or ["toolboxv2", "src", "lib", "docs"]
        self.exclude_dirs = exclude_dirs or [
            "__pycache__", ".git", "node_modules", ".venv", "venv", "env",
            ".pytest_cache", ".mypy_cache", "dist", "build", ".tox",
            "coverage_html_report", ".coverage", ".next", ".nuxt",
            "target", "bin", "obj", ".gradle", ".idea", ".vscode"
        ]

    async def update_index_precise(self, current_index: DocsIndex,
                                   force_full_scan: bool = False,
                                   max_files_per_batch: int = 10) -> Tuple[DocsIndex, List[str], Dict[str, List[str]]]:
        """Update index with precise section-level tracking"""
        logger.info("Starting precise index update...")

        update_notes = []
        section_changes = {}  # file_path -> [changed_section_ids]

        if force_full_scan:
            return await self._full_scan_with_sections(current_index, max_files_per_batch)

        # Quick git change detection with timeout
        try:
            changes = await asyncio.wait_for(
                asyncio.get_event_loop().run_in_executor(
                    None, self.git_detector.get_changed_files, current_index.last_git_commit
                ),
                timeout=15.0
            )
        except asyncio.TimeoutError:
            logger.warning("Git detection timed out, using cached index")
            return current_index, ["Git timeout - using cached index"], {}

        # Process changes in batches
        changed_files = [c for c in changes if self._should_include_file(Path(c.file_path))]

        for i in range(0, len(changed_files), max_files_per_batch):
            batch = changed_files[i:i + max_files_per_batch]

            for change in batch:
                file_path = Path(change.file_path)

                if change.change_type == ChangeType.DELETED:
                    removed_sections = self._remove_file_from_index(current_index, str(file_path))
                    update_notes.append(f"Removed file: {file_path} ({len(removed_sections)} sections)")

                elif change.change_type in [ChangeType.ADDED, ChangeType.MODIFIED]:
                    if not file_path.exists():
                        continue

                    # Quick hash check first
                    new_hash = self._get_file_hash(file_path)
                    old_hash = current_index.file_hashes.get(str(file_path))

                    if new_hash == old_hash:
                        continue  # No actual change

                    # Section-level update for markdown files
                    if file_path.suffix == '.md' and file_path.is_relative_to(self.docs_root):
                        changed_section_ids = await self._update_markdown_sections(
                            current_index, file_path
                        )
                        if changed_section_ids:
                            section_changes[str(file_path)] = changed_section_ids
                            update_notes.append(f"Updated {len(changed_section_ids)} sections in {file_path.name}")

                    # Code files - update entire file (faster for code)
                    elif file_path.suffix in ['.py', '.js', '.ts', '.jsx', '.tsx']:
                        self._update_file_in_index(current_index, file_path)
                        update_notes.append(f"Updated code file: {file_path}")

                    # Update file hash
                    current_index.file_hashes[str(file_path)] = new_hash

            # Yield control between batches
            await asyncio.sleep(0.01)

        # Update timestamps
        current_index.last_git_commit = self.git_detector.get_current_commit_hash()
        current_index.last_indexed = datetime.now()

        return current_index, update_notes, section_changes

    async def _update_markdown_sections(self, index: DocsIndex, file_path: Path) -> List[str]:
        """Update only changed sections in a markdown file"""
        try:
            # Get existing sections for this file
            existing_sections = {
                sid: section for sid, section in index.sections.items()
                if section.file_path == str(file_path)
            }

            # Parse with change detection
            new_sections, changed_section_ids = self.md_parser.parse_file_incremental(
                file_path, existing_sections
            )

            # Update only changed sections
            for section in new_sections:
                if section.change_detected or section.section_id not in index.sections:
                    index.sections[section.section_id] = section
                    # Update section hash tracking
                    index.section_hashes[section.section_id] = section.hash_signature

            # Remove sections that no longer exist
            current_section_ids = {s.section_id for s in new_sections}
            to_remove = [sid for sid in existing_sections.keys() if sid not in current_section_ids]

            for sid in to_remove:
                if sid in index.sections:
                    del index.sections[sid]
                if sid in index.section_hashes:
                    del index.section_hashes[sid]
                changed_section_ids.append(f"REMOVED:{sid}")

            return changed_section_ids

        except Exception as e:
            logger.error(f"Error updating markdown sections in {file_path}: {e}")
            return []

    def build_initial_index(self, file_extensions: List[str] = None) -> DocsIndex:
        """Build complete index for the first time with directory filtering"""
        logger.info("Building initial documentation index...")

        if file_extensions is None:
            file_extensions = ['.py', '.js', '.ts', '.jsx', '.tsx', '.md']

        index = DocsIndex()

        # Get current git commit
        index.last_git_commit = self.git_detector.get_current_commit_hash()

        # Get filtered file list
        target_files = self._get_filtered_files(file_extensions)

        logger.info(f"Processing {len(target_files)} files in {len(self.include_dirs)} directories")

        # Process each file type
        for file_path in target_files:
            logger.debug(f"Indexing {file_path}")
            try:
                if file_path.suffix == '.py':
                    self._index_python_file(file_path, index)
                elif file_path.suffix in ['.js', '.ts', '.jsx', '.tsx']:
                    self._index_js_file(file_path, index)
                elif file_path.suffix == '.md' and file_path.is_relative_to(self.docs_root):
                    self._index_md_file(file_path, index)

                # Store file hash for change detection
                index.file_hashes[str(file_path)] = self._get_file_hash(file_path)

            except Exception as e:
                logger.error(f"Error indexing {file_path}: {e}")
                continue

        index.last_indexed = datetime.now()

        logger.info(f"Initial index built: {len(index.code_elements)} code elements, "
                    f"{len(index.sections)} doc sections, {len(index.import_refs)} import maps")

        return index

    def update_index(self, current_index: DocsIndex,
                     force_full_scan: bool = False) -> Tuple[DocsIndex, List[str]]:
        """Update index based on detected changes or force full scan"""
        logger.info("Updating documentation index...")

        update_notes = []

        if force_full_scan:
            logger.info("Performing force full scan...")
            # Get all current files
            current_files = set(self._get_filtered_files(['.py', '.js', '.ts', '.jsx', '.tsx', '.md']))

            # Check each file for changes
            for file_path in current_files:
                new_hash = self._get_file_hash(file_path)
                old_hash = current_index.file_hashes.get(str(file_path))

                if new_hash != old_hash:
                    self._update_file_in_index(current_index, file_path)
                    update_notes.append(f"Updated (force scan): {file_path}")
                    current_index.file_hashes[str(file_path)] = new_hash

            # Remove files that no longer exist
            existing_files = {str(f) for f in current_files}
            to_remove = [f for f in current_index.file_hashes.keys() if f not in existing_files]

            for file_path in to_remove:
                self._remove_file_from_index(current_index, file_path)
                update_notes.append(f"Removed (no longer exists): {file_path}")

        else:
            # Git-based change detection
            changes = self.git_detector.get_changed_files(current_index.last_git_commit)

            for change in changes:
                file_path = Path(change.file_path)

                # Skip if not in our target directories
                if not self._should_include_file(file_path):
                    continue

                if change.change_type == ChangeType.DELETED:
                    self._remove_file_from_index(current_index, str(file_path))
                    update_notes.append(f"Removed (git): {file_path}")

                elif change.change_type == ChangeType.RENAMED:
                    if change.old_path:
                        self._remove_file_from_index(current_index, change.old_path)
                        update_notes.append(f"Removed (renamed from): {change.old_path}")

                    if file_path.exists():
                        self._update_file_in_index(current_index, file_path)
                        update_notes.append(f"Added (renamed to): {file_path}")
                        current_index.file_hashes[str(file_path)] = self._get_file_hash(file_path)

                elif change.change_type in [ChangeType.ADDED, ChangeType.MODIFIED]:
                    if not file_path.exists():
                        continue

                    # Verify actual change with hash comparison
                    new_hash = self._get_file_hash(file_path)
                    old_hash = current_index.file_hashes.get(str(file_path))

                    if new_hash != old_hash:
                        self._update_file_in_index(current_index, file_path)
                        update_notes.append(f"Updated (git {change.change_type.value}): {file_path}")
                        current_index.file_hashes[str(file_path)] = new_hash

        # Update git commit and timestamp
        current_index.last_git_commit = self.git_detector.get_current_commit_hash()
        current_index.last_indexed = datetime.now()

        if update_notes:
            logger.info(f"Index updated with {len(update_notes)} changes")
        else:
            logger.info("No changes detected in index update")

        return current_index, update_notes

    def _get_filtered_files(self, extensions: List[str]) -> List[Path]:
        """Get list of files matching extensions and directory filters"""
        files = []

        # If include_dirs is specified, only search in those directories
        search_dirs = []
        for include_dir in self.include_dirs:
            search_path = self.project_root / include_dir
            if search_path.exists() and search_path.is_dir():
                search_dirs.append(search_path)

        # If no include dirs exist, search entire project root
        if not search_dirs:
            search_dirs = [self.project_root]

        for search_dir in search_dirs:
            for ext in extensions:
                pattern = f"**/*{ext}"
                for file_path in search_dir.rglob(pattern):
                    if self._should_include_file(file_path):
                        files.append(file_path)

        return list(set(files))  # Remove duplicates

    def _should_include_file(self, file_path: Path) -> bool:
        """Check if file should be included based on directory filters"""
        file_str = str(file_path)

        # Check exclude patterns
        for exclude_dir in self.exclude_dirs:
            if exclude_dir in file_str:
                return False

        # If include_dirs specified, file must be in one of them
        if self.include_dirs:
            for include_dir in self.include_dirs:
                include_path = self.project_root / include_dir
                try:
                    if file_path.is_relative_to(include_path):
                        return True
                except ValueError:
                    continue
            return False

        return True

    def _update_file_in_index(self, index: DocsIndex, file_path: Path):
        """Update index for a specific file"""
        # Remove old entries first
        self._remove_file_from_index(index, str(file_path))

        # Add new entries based on file type
        try:
            if file_path.suffix == '.py':
                self._index_python_file(file_path, index)
            elif file_path.suffix in ['.js', '.ts', '.jsx', '.tsx']:
                self._index_js_file(file_path, index)
            elif file_path.suffix == '.md' and file_path.is_relative_to(self.docs_root):
                self._index_md_file(file_path, index)
        except Exception as e:
            logger.error(f"Error updating file {file_path}: {e}")

    def _index_python_file(self, file_path: Path, index: DocsIndex):
        """Index a Python file"""
        # Extract code elements
        elements = self.code_extractor.extract_python_elements(file_path)
        for element in elements:
            element_id = f"{element.file_path}:{element.name}"
            if element.parent_class:
                element_id = f"{element.file_path}:{element.parent_class}.{element.name}"
            index.code_elements[element_id] = element

        # Analyze imports
        imports = self.import_analyzer.analyze_python_imports(file_path)
        if imports:
            index.import_refs[str(file_path)] = imports

    def _index_js_file(self, file_path: Path, index: DocsIndex):
        """Index a JavaScript/TypeScript file"""
        imports = self.import_analyzer.analyze_js_imports(file_path)
        if imports:
            index.import_refs[str(file_path)] = imports

    def _index_md_file(self, file_path: Path, index: DocsIndex):
        """Index a markdown file"""
        sections = self.md_parser.parse_file(file_path)
        for section in sections:
            index.sections[section.section_id] = section

    def _get_file_hash(self, file_path: Path) -> str:
        """Get MD5 hash of file content"""
        try:
            with open(file_path, 'rb') as f:
                return hashlib.md5(f.read()).hexdigest()
        except Exception:
            return ""

    def _remove_file_from_index(self, index: DocsIndex, file_path: str):
        """Remove all references to a file from index"""
        # Remove code elements
        to_remove = [k for k, v in index.code_elements.items() if v.file_path == file_path]
        for key in to_remove:
            del index.code_elements[key]

        # Remove doc sections
        to_remove = [k for k, v in index.sections.items() if v.file_path == file_path]
        for key in to_remove:
            del index.sections[key]

        # Remove import refs
        if file_path in index.import_refs:
            del index.import_refs[file_path]

        # Remove file hash
        if file_path in index.file_hashes:
            del index.file_hashes[file_path]

build_initial_index(file_extensions=None)

Build complete index for the first time with directory filtering

Source code in toolboxv2/utils/extras/mkdocs.py

def build_initial_index(self, file_extensions: List[str] = None) -> DocsIndex:
    """Build complete index for the first time with directory filtering"""
    logger.info("Building initial documentation index...")

    if file_extensions is None:
        file_extensions = ['.py', '.js', '.ts', '.jsx', '.tsx', '.md']

    index = DocsIndex()

    # Get current git commit
    index.last_git_commit = self.git_detector.get_current_commit_hash()

    # Get filtered file list
    target_files = self._get_filtered_files(file_extensions)

    logger.info(f"Processing {len(target_files)} files in {len(self.include_dirs)} directories")

    # Process each file type
    for file_path in target_files:
        logger.debug(f"Indexing {file_path}")
        try:
            if file_path.suffix == '.py':
                self._index_python_file(file_path, index)
            elif file_path.suffix in ['.js', '.ts', '.jsx', '.tsx']:
                self._index_js_file(file_path, index)
            elif file_path.suffix == '.md' and file_path.is_relative_to(self.docs_root):
                self._index_md_file(file_path, index)

            # Store file hash for change detection
            index.file_hashes[str(file_path)] = self._get_file_hash(file_path)

        except Exception as e:
            logger.error(f"Error indexing {file_path}: {e}")
            continue

    index.last_indexed = datetime.now()

    logger.info(f"Initial index built: {len(index.code_elements)} code elements, "
                f"{len(index.sections)} doc sections, {len(index.import_refs)} import maps")

    return index

update_index(current_index, force_full_scan=False)

Update index based on detected changes or force full scan

Source code in toolboxv2/utils/extras/mkdocs.py

def update_index(self, current_index: DocsIndex,
                 force_full_scan: bool = False) -> Tuple[DocsIndex, List[str]]:
    """Update index based on detected changes or force full scan"""
    logger.info("Updating documentation index...")

    update_notes = []

    if force_full_scan:
        logger.info("Performing force full scan...")
        # Get all current files
        current_files = set(self._get_filtered_files(['.py', '.js', '.ts', '.jsx', '.tsx', '.md']))

        # Check each file for changes
        for file_path in current_files:
            new_hash = self._get_file_hash(file_path)
            old_hash = current_index.file_hashes.get(str(file_path))

            if new_hash != old_hash:
                self._update_file_in_index(current_index, file_path)
                update_notes.append(f"Updated (force scan): {file_path}")
                current_index.file_hashes[str(file_path)] = new_hash

        # Remove files that no longer exist
        existing_files = {str(f) for f in current_files}
        to_remove = [f for f in current_index.file_hashes.keys() if f not in existing_files]

        for file_path in to_remove:
            self._remove_file_from_index(current_index, file_path)
            update_notes.append(f"Removed (no longer exists): {file_path}")

    else:
        # Git-based change detection
        changes = self.git_detector.get_changed_files(current_index.last_git_commit)

        for change in changes:
            file_path = Path(change.file_path)

            # Skip if not in our target directories
            if not self._should_include_file(file_path):
                continue

            if change.change_type == ChangeType.DELETED:
                self._remove_file_from_index(current_index, str(file_path))
                update_notes.append(f"Removed (git): {file_path}")

            elif change.change_type == ChangeType.RENAMED:
                if change.old_path:
                    self._remove_file_from_index(current_index, change.old_path)
                    update_notes.append(f"Removed (renamed from): {change.old_path}")

                if file_path.exists():
                    self._update_file_in_index(current_index, file_path)
                    update_notes.append(f"Added (renamed to): {file_path}")
                    current_index.file_hashes[str(file_path)] = self._get_file_hash(file_path)

            elif change.change_type in [ChangeType.ADDED, ChangeType.MODIFIED]:
                if not file_path.exists():
                    continue

                # Verify actual change with hash comparison
                new_hash = self._get_file_hash(file_path)
                old_hash = current_index.file_hashes.get(str(file_path))

                if new_hash != old_hash:
                    self._update_file_in_index(current_index, file_path)
                    update_notes.append(f"Updated (git {change.change_type.value}): {file_path}")
                    current_index.file_hashes[str(file_path)] = new_hash

    # Update git commit and timestamp
    current_index.last_git_commit = self.git_detector.get_current_commit_hash()
    current_index.last_indexed = datetime.now()

    if update_notes:
        logger.info(f"Index updated with {len(update_notes)} changes")
    else:
        logger.info("No changes detected in index update")

    return current_index, update_notes

update_index_precise(current_index, force_full_scan=False, max_files_per_batch=10) async

Update index with precise section-level tracking

Source code in toolboxv2/utils/extras/mkdocs.py

async def update_index_precise(self, current_index: DocsIndex,
                               force_full_scan: bool = False,
                               max_files_per_batch: int = 10) -> Tuple[DocsIndex, List[str], Dict[str, List[str]]]:
    """Update index with precise section-level tracking"""
    logger.info("Starting precise index update...")

    update_notes = []
    section_changes = {}  # file_path -> [changed_section_ids]

    if force_full_scan:
        return await self._full_scan_with_sections(current_index, max_files_per_batch)

    # Quick git change detection with timeout
    try:
        changes = await asyncio.wait_for(
            asyncio.get_event_loop().run_in_executor(
                None, self.git_detector.get_changed_files, current_index.last_git_commit
            ),
            timeout=15.0
        )
    except asyncio.TimeoutError:
        logger.warning("Git detection timed out, using cached index")
        return current_index, ["Git timeout - using cached index"], {}

    # Process changes in batches
    changed_files = [c for c in changes if self._should_include_file(Path(c.file_path))]

    for i in range(0, len(changed_files), max_files_per_batch):
        batch = changed_files[i:i + max_files_per_batch]

        for change in batch:
            file_path = Path(change.file_path)

            if change.change_type == ChangeType.DELETED:
                removed_sections = self._remove_file_from_index(current_index, str(file_path))
                update_notes.append(f"Removed file: {file_path} ({len(removed_sections)} sections)")

            elif change.change_type in [ChangeType.ADDED, ChangeType.MODIFIED]:
                if not file_path.exists():
                    continue

                # Quick hash check first
                new_hash = self._get_file_hash(file_path)
                old_hash = current_index.file_hashes.get(str(file_path))

                if new_hash == old_hash:
                    continue  # No actual change

                # Section-level update for markdown files
                if file_path.suffix == '.md' and file_path.is_relative_to(self.docs_root):
                    changed_section_ids = await self._update_markdown_sections(
                        current_index, file_path
                    )
                    if changed_section_ids:
                        section_changes[str(file_path)] = changed_section_ids
                        update_notes.append(f"Updated {len(changed_section_ids)} sections in {file_path.name}")

                # Code files - update entire file (faster for code)
                elif file_path.suffix in ['.py', '.js', '.ts', '.jsx', '.tsx']:
                    self._update_file_in_index(current_index, file_path)
                    update_notes.append(f"Updated code file: {file_path}")

                # Update file hash
                current_index.file_hashes[str(file_path)] = new_hash

        # Yield control between batches
        await asyncio.sleep(0.01)

    # Update timestamps
    current_index.last_git_commit = self.git_detector.get_current_commit_hash()
    current_index.last_indexed = datetime.now()

    return current_index, update_notes, section_changes

FileChange dataclass

Represents a file change detected by git

Source code in toolboxv2/utils/extras/mkdocs.py

@dataclass
class FileChange:
    """Represents a file change detected by git"""
    file_path: str
    change_type: ChangeType
    old_hash: Optional[str] = None
    new_hash: Optional[str] = None
    old_path: Optional[str] = None  # for renamed files

GitChangeDetector

Detects changes in the repository since last index update

Source code in toolboxv2/utils/extras/mkdocs.py

class GitChangeDetector:
    """Detects changes in the repository since last index update"""

    def __init__(self, repo_root: Path):
        self.repo_root = repo_root

    def get_current_commit_hash(self) -> Optional[str]:
        """Get current git commit hash"""
        try:
            result = subprocess.run(
                ["git", "rev-parse", "HEAD"],
                cwd=self.repo_root,
                capture_output=True,
                text=True
            )
            return result.stdout.strip() if result.returncode == 0 else None
        except Exception:
            return None

    def get_changed_files(self, since_commit: Optional[str] = None) -> List[FileChange]:
        """Get list of changed files since given commit with timeout"""
        changes = []

        try:
            if since_commit:
                cmd = ["git", "diff", "--name-status", f"{since_commit}..HEAD"]
            else:
                cmd = ["git", "ls-files"]

            # Add timeout to subprocess
            result = subprocess.run(
                cmd,
                cwd=self.repo_root,
                capture_output=True,
                text=True,
                timeout=20  # 20 second timeout for git operations
            )

            if result.returncode != 0:
                logger.warning(f"Git command failed with return code {result.returncode}")
                return changes

            # Process output with limits
            lines = result.stdout.strip().split('\n')[:1000]  # Limit to 1000 files

            for line in lines:
                if not line:
                    continue

                if since_commit:
                    parts = line.split('\t')
                    if len(parts) >= 2:
                        status = parts[0]
                        file_path = parts[1]

                        if status == 'A':
                            change_type = ChangeType.ADDED
                        elif status == 'M':
                            change_type = ChangeType.MODIFIED
                        elif status == 'D':
                            change_type = ChangeType.DELETED
                        elif status.startswith('R'):
                            change_type = ChangeType.RENAMED
                            old_path = parts[1] if len(parts) > 2 else None
                            file_path = parts[2] if len(parts) > 2 else parts[1]
                        else:
                            continue

                        changes.append(FileChange(
                            file_path=file_path,
                            change_type=change_type,
                            old_path=old_path if change_type == ChangeType.RENAMED else None
                        ))
                else:
                    changes.append(FileChange(
                        file_path=line.strip(),
                        change_type=ChangeType.ADDED
                    ))

        except subprocess.TimeoutExpired:
            logger.error("Git operation timed out after 20 seconds")
        except Exception as e:
            logger.error(f"Error detecting git changes: {e}")

        return changes

get_changed_files(since_commit=None)

Get list of changed files since given commit with timeout

Source code in toolboxv2/utils/extras/mkdocs.py

def get_changed_files(self, since_commit: Optional[str] = None) -> List[FileChange]:
    """Get list of changed files since given commit with timeout"""
    changes = []

    try:
        if since_commit:
            cmd = ["git", "diff", "--name-status", f"{since_commit}..HEAD"]
        else:
            cmd = ["git", "ls-files"]

        # Add timeout to subprocess
        result = subprocess.run(
            cmd,
            cwd=self.repo_root,
            capture_output=True,
            text=True,
            timeout=20  # 20 second timeout for git operations
        )

        if result.returncode != 0:
            logger.warning(f"Git command failed with return code {result.returncode}")
            return changes

        # Process output with limits
        lines = result.stdout.strip().split('\n')[:1000]  # Limit to 1000 files

        for line in lines:
            if not line:
                continue

            if since_commit:
                parts = line.split('\t')
                if len(parts) >= 2:
                    status = parts[0]
                    file_path = parts[1]

                    if status == 'A':
                        change_type = ChangeType.ADDED
                    elif status == 'M':
                        change_type = ChangeType.MODIFIED
                    elif status == 'D':
                        change_type = ChangeType.DELETED
                    elif status.startswith('R'):
                        change_type = ChangeType.RENAMED
                        old_path = parts[1] if len(parts) > 2 else None
                        file_path = parts[2] if len(parts) > 2 else parts[1]
                    else:
                        continue

                    changes.append(FileChange(
                        file_path=file_path,
                        change_type=change_type,
                        old_path=old_path if change_type == ChangeType.RENAMED else None
                    ))
            else:
                changes.append(FileChange(
                    file_path=line.strip(),
                    change_type=ChangeType.ADDED
                ))

    except subprocess.TimeoutExpired:
        logger.error("Git operation timed out after 20 seconds")
    except Exception as e:
        logger.error(f"Error detecting git changes: {e}")

    return changes

get_current_commit_hash()

Get current git commit hash

Source code in toolboxv2/utils/extras/mkdocs.py

def get_current_commit_hash(self) -> Optional[str]:
    """Get current git commit hash"""
    try:
        result = subprocess.run(
            ["git", "rev-parse", "HEAD"],
            cwd=self.repo_root,
            capture_output=True,
            text=True
        )
        return result.stdout.strip() if result.returncode == 0 else None
    except Exception:
        return None

ImportAnalyzer

Analyzes imports and dependencies in Python and JS files

Source code in toolboxv2/utils/extras/mkdocs.py

class ImportAnalyzer:
    """Analyzes imports and dependencies in Python and JS files"""

    def __init__(self, project_root: Path):
        self.project_root = project_root

    def analyze_python_imports(self, file_path: Path) -> List[ImportReference]:
        """Analyze Python imports"""
        imports = []

        try:
            with open(file_path, 'r', encoding='utf-8') as f:
                content = f.read()

            tree = ast.parse(content)

            for node in ast.walk(tree):
                if isinstance(node, ast.Import):
                    for alias in node.names:
                        imports.append(ImportReference(
                            source_file=str(file_path),
                            target_file=self._resolve_python_import(alias.name),
                            import_name=alias.name,
                            line_number=node.lineno,
                            import_type='python'
                        ))

                elif isinstance(node, ast.ImportFrom):
                    module = node.module or ""
                    for alias in node.names:
                        target_file = self._resolve_python_import(f"{module}.{alias.name}")
                        imports.append(ImportReference(
                            source_file=str(file_path),
                            target_file=target_file,
                            import_name=f"{module}.{alias.name}",
                            line_number=node.lineno,
                            import_type='relative' if node.level > 0 else 'absolute'
                        ))

        except Exception as e:
            logger.error(f"Error analyzing Python imports in {file_path}: {e}")

        return imports

    def analyze_js_imports(self, file_path: Path) -> List[ImportReference]:
        """Analyze JavaScript/TypeScript imports"""
        imports = []

        try:
            with open(file_path, 'r', encoding='utf-8') as f:
                content = f.read()

            # Regex patterns for different import types
            patterns = [
                r'import\s+(?:{\s*([^}]+)\s*}|\*\s+as\s+(\w+)|(\w+))\s+from\s+["\']([^"\']+)["\']',
                r'const\s+(?:{\s*([^}]+)\s*}|(\w+))\s*=\s*require\(["\']([^"\']+)["\']\)',
                r'import\(["\']([^"\']+)["\']\)'
            ]

            for i, line in enumerate(content.split('\n'), 1):
                for pattern in patterns:
                    matches = re.finditer(pattern, line)
                    for match in matches:
                        groups = match.groups()
                        target = groups[-1]  # Last group is always the module path

                        imports.append(ImportReference(
                            source_file=str(file_path),
                            target_file=self._resolve_js_import(file_path.parent, target),
                            import_name=target,
                            line_number=i,
                            import_type='js'
                        ))

        except Exception as e:
            logger.error(f"Error analyzing JS imports in {file_path}: {e}")

        return imports

    def _resolve_python_import(self, import_name: str) -> str:
        """Resolve Python import to file path"""
        # Try to find the actual file
        parts = import_name.split('.')

        # Check in project root
        potential_paths = [
            self.project_root / '/'.join(parts) / '__init__.py',
            self.project_root / f"{'/'.join(parts)}.py",
        ]

        for path in potential_paths:
            if path.exists():
                return str(path)

        return import_name  # Return original if not found

    def _resolve_js_import(self, current_dir: Path, import_path: str) -> str:
        """Resolve JS import to file path"""
        if import_path.startswith('.'):
            # Relative import
            resolved = (current_dir / import_path).resolve()

            # Try different extensions
            for ext in ['.js', '.ts', '.jsx', '.tsx', '.json']:
                if resolved.with_suffix(ext).exists():
                    return str(resolved.with_suffix(ext))

            # Try index files
            for ext in ['.js', '.ts']:
                index_file = resolved / f"index{ext}"
                if index_file.exists():
                    return str(index_file)

        return import_path  # Return original if not found

analyze_js_imports(file_path)

Analyze JavaScript/TypeScript imports

Source code in toolboxv2/utils/extras/mkdocs.py

def analyze_js_imports(self, file_path: Path) -> List[ImportReference]:
    """Analyze JavaScript/TypeScript imports"""
    imports = []

    try:
        with open(file_path, 'r', encoding='utf-8') as f:
            content = f.read()

        # Regex patterns for different import types
        patterns = [
            r'import\s+(?:{\s*([^}]+)\s*}|\*\s+as\s+(\w+)|(\w+))\s+from\s+["\']([^"\']+)["\']',
            r'const\s+(?:{\s*([^}]+)\s*}|(\w+))\s*=\s*require\(["\']([^"\']+)["\']\)',
            r'import\(["\']([^"\']+)["\']\)'
        ]

        for i, line in enumerate(content.split('\n'), 1):
            for pattern in patterns:
                matches = re.finditer(pattern, line)
                for match in matches:
                    groups = match.groups()
                    target = groups[-1]  # Last group is always the module path

                    imports.append(ImportReference(
                        source_file=str(file_path),
                        target_file=self._resolve_js_import(file_path.parent, target),
                        import_name=target,
                        line_number=i,
                        import_type='js'
                    ))

    except Exception as e:
        logger.error(f"Error analyzing JS imports in {file_path}: {e}")

    return imports

analyze_python_imports(file_path)

Analyze Python imports

Source code in toolboxv2/utils/extras/mkdocs.py

def analyze_python_imports(self, file_path: Path) -> List[ImportReference]:
    """Analyze Python imports"""
    imports = []

    try:
        with open(file_path, 'r', encoding='utf-8') as f:
            content = f.read()

        tree = ast.parse(content)

        for node in ast.walk(tree):
            if isinstance(node, ast.Import):
                for alias in node.names:
                    imports.append(ImportReference(
                        source_file=str(file_path),
                        target_file=self._resolve_python_import(alias.name),
                        import_name=alias.name,
                        line_number=node.lineno,
                        import_type='python'
                    ))

            elif isinstance(node, ast.ImportFrom):
                module = node.module or ""
                for alias in node.names:
                    target_file = self._resolve_python_import(f"{module}.{alias.name}")
                    imports.append(ImportReference(
                        source_file=str(file_path),
                        target_file=target_file,
                        import_name=f"{module}.{alias.name}",
                        line_number=node.lineno,
                        import_type='relative' if node.level > 0 else 'absolute'
                    ))

    except Exception as e:
        logger.error(f"Error analyzing Python imports in {file_path}: {e}")

    return imports

ImportReference dataclass

Represents an import/dependency reference

Source code in toolboxv2/utils/extras/mkdocs.py

@dataclass
class ImportReference:
    """Represents an import/dependency reference"""
    source_file: str
    target_file: str
    import_name: str
    line_number: int
    import_type: str  # 'python', 'js', 'relative', 'absolute'

MarkdownDocsSystem

Production-ready unified markdown documentation system

Source code in toolboxv2/utils/extras/mkdocs.py

class MarkdownDocsSystem:
    """Production-ready unified markdown documentation system"""

    def __init__(self, app: AppType, docs_root: str = "../docs", source_root: str = ".",
                 include_dirs: List[str] = None, exclude_dirs: List[str] = None):
        self.app = app
        self.docs_root = Path(docs_root)
        self.source_root = Path(source_root)
        self.project_root = Path.cwd()

        # Directory filters
        self.include_dirs = include_dirs or ["toolboxv2", "flows", "mods", "utils", "tbjs", "tests", "tcm", "docs"]
        self.exclude_dirs = exclude_dirs or [
            "__pycache__", ".git", "node_modules", ".venv", "venv", "env", "python_env",
            ".pytest_cache", ".mypy_cache", "dist", "build", ".tox", "coverage_html_report",
            ".coverage", ".next", ".nuxt", "target", "bin", "obj", ".gradle", ".idea",
            ".vscode", "temp", "tmp", "logs", ".cache", "coverage", ".data", ".config",
            ".info", "web", "simple-core", "src-core"
        ]

        # Index management
        self.index_file = self.docs_root / '.docs_index.json'
        self.current_index: Optional[DocsIndex] = None

        # Ensure directories exist
        self.docs_root.mkdir(exist_ok=True)

        # Internal cache for performance
        self._search_cache = {}
        self._cache_timeout = 300  # 5 minutes

    # ==================== CORE INDEX MANAGEMENT ====================


    def _load_index(self, minimal: bool = False, force_reload: bool = False) -> DocsIndex:
        """Unified index loading with proper incremental loading"""
        # Return cached index if available and not forcing reload
        if not force_reload and self.current_index and not minimal:
            return self.current_index

        if not self.index_file.exists():
            logger.info("No index file found, will build new index")
            return DocsIndex()

        try:
            with open(self.index_file, 'r', encoding='utf-8') as f:
                data = json.load(f)

            index = DocsIndex()
            index.last_indexed = datetime.fromisoformat(
                data.get('last_indexed', datetime.now().isoformat())
            )
            index.last_git_commit = data.get('last_git_commit')
            index.version = data.get('version', '1.1')
            index.file_hashes = data.get('file_hashes', {})
            index.section_hashes = data.get('section_hashes', {})

            # Load sections with optional truncation for performance
            sections_data = data.get('sections', {})
            section_limit = 200 if minimal else None
            section_count = 0

            for section_id, section_data in sections_data.items():
                if minimal and section_count >= section_limit:
                    break

                content = section_data['content']
                if minimal:
                    content = content[:800]  # Truncate for speed

                index.sections[section_id] = DocSection(
                    section_id=section_data['section_id'],
                    file_path=section_data['file_path'],
                    title=section_data['title'],
                    content=content,
                    level=section_data['level'],
                    line_start=section_data['line_start'],
                    line_end=section_data['line_end'],
                    source_refs=section_data.get('source_refs', [])[:5 if minimal else None],
                    tags=section_data.get('tags', [])[:3 if minimal else None],
                    hash_signature=section_data.get('hash_signature', ''),
                    content_hash=section_data.get('content_hash', ''),
                    last_modified=datetime.fromisoformat(
                        section_data.get('last_modified', datetime.now().isoformat()))
                )
                section_count += 1

            # Load code elements only if not minimal
            if not minimal:
                for element_id, element_data in data.get('code_elements', {}).items():
                    index.code_elements[element_id] = CodeElement(
                        name=element_data['name'],
                        element_type=element_data['element_type'],
                        file_path=element_data['file_path'],
                        line_start=element_data['line_start'],
                        line_end=element_data['line_end'],
                        signature=element_data['signature'],
                        docstring=element_data.get('docstring'),
                        hash_signature=element_data.get('hash_signature', ''),
                        parent_class=element_data.get('parent_class')
                    )

            # Cache full index for future use
            if not minimal:
                self.current_index = index

            logger.info(
                f"Loaded {'minimal' if minimal else 'full'} index: {len(index.sections)} sections, {len(index.code_elements)} elements")
            return index

        except Exception as e:
            logger.error(f"Error loading index: {e}")
            return DocsIndex()

    def _save_index(self, index: DocsIndex):
        """Optimized index saving"""
        try:
            data = {
                'version': index.version,
                'last_git_commit': index.last_git_commit,
                'last_indexed': index.last_indexed.isoformat(),
                'file_hashes': index.file_hashes,
                'section_hashes': index.section_hashes,
                'sections': {},
                'code_elements': {}
            }

            # Convert sections efficiently
            for section_id, section in index.sections.items():
                section_dict = asdict(section)
                section_dict['last_modified'] = section.last_modified.isoformat()
                data['sections'][section_id] = section_dict

            # Convert code elements efficiently
            for element_id, element in index.code_elements.items():
                data['code_elements'][element_id] = asdict(element)

            # Write with minimal formatting for speed
            with open(self.index_file, 'w', encoding='utf-8') as f:
                json.dump(data, f, separators=(',', ':'), ensure_ascii=False)

        except Exception as e:
            logger.error(f"Error saving index: {e}")

    async def _save_index_async(self, index: DocsIndex):
        """Async wrapper for index saving"""
        try:
            await asyncio.get_event_loop().run_in_executor(None, self._save_index, index)
        except Exception as e:
            logger.error(f"Async index save failed: {e}")

    # ==================== CORE FUNCTIONALITY ====================

    async def docs_reader(self,
                          query: Optional[str] = None,
                          section_id: Optional[str] = None,
                          file_path: Optional[str] = None,
                          tags: Optional[List[str]] = None,
                          include_source_refs: bool = True,
                          format_type: str = "structured",
                          max_results: int = 25) -> Result:
        """Ultra-fast unified docs reader with proper index loading"""
        try:
            start_time = time.time()

            # Load index if needed - try cached first, then saved, then build
            if not self.current_index:
                # First try to load from saved file
                self.current_index = self._load_index(minimal=False)

                # If no sections found, suggest running initial_docs_parse
                if not self.current_index.sections:
                    if self.index_file.exists():
                        logger.warning("Index file exists but contains no documentation sections")
                        # Try to find any markdown files
                        md_files = list(self.docs_root.rglob('*.md'))
                        if md_files:
                            logger.info(f"Found {len(md_files)} markdown files, re-parsing...")
                            # Re-parse markdown files
                            for md_file in md_files[:10]:  # Limit for quick check
                                sections = self._parse_markdown_file(md_file)
                                for section in sections:
                                    self.current_index.sections[section.section_id] = section

                            if self.current_index.sections:
                                logger.info(f"Re-parsed {len(self.current_index.sections)} sections")
                                await self._save_index_async(self.current_index)

                    if not self.current_index.sections:
                        return Result.default_user_error(
                            "No documentation sections found. Run initial_docs_parse() to build the documentation index, "
                            "or create some .md files in the docs directory."
                        )

            # Check cache for repeated queries
            cache_key = f"{query}:{section_id}:{file_path}:{tags}:{format_type}:{max_results}"
            if cache_key in self._search_cache:
                cache_entry = self._search_cache[cache_key]
                if time.time() - cache_entry['timestamp'] < self._cache_timeout:
                    return Result.ok(cache_entry['result'])

            # Fast path for specific section
            if section_id:
                if section_id in self.current_index.sections:
                    section = self.current_index.sections[section_id]
                    result = self._format_single_section(section, include_source_refs, format_type)
                    return result
                else:
                    return Result.default_user_error(f"Section not found: {section_id}")

            # Search and format results
            matching_sections = await self._search_sections(query, file_path, tags, max_results, start_time)
            result_data = self._format_sections(matching_sections, include_source_refs, format_type)

            # Cache the result
            self._search_cache[cache_key] = {
                'result': result_data,
                'timestamp': time.time()
            }

            return Result.ok(result_data)

        except Exception as e:
            logger.error(f"Error in docs_reader: {e}")
            return Result.default_user_error(f"Error reading documentation: {e}")

    async def docs_writer(self,
                          action: str,
                          file_path: Optional[str] = None,
                          section_title: Optional[str] = None,
                          content: Optional[str] = None,
                          source_file: Optional[str] = None,
                          auto_generate: bool = False,
                          position: Optional[str] = None,
                          level: int = 2) -> Result:
        """Unified optimized docs writer"""
        try:
            if not self.current_index:
                self.current_index = self._load_index(minimal=True)

            result = {"action": action, "timestamp": datetime.now().isoformat()}

            if action == "update_section":
                return await self._update_section(file_path, section_title, content,
                                                  source_file, auto_generate)
            elif action == "add_section":
                return await self._add_section(file_path, section_title, content,
                                               source_file, auto_generate, position, level)
            elif action == "create_file":
                return await self._create_file(file_path, content, source_file, auto_generate)
            elif action == "generate_from_code":
                return await self._generate_from_code(source_file, file_path, auto_generate)
            else:
                return Result.default_user_error(f"Unknown action: {action}")

        except Exception as e:
            logger.error(f"Error in docs_writer: {e}")
            return Result.default_user_error(f"Error writing documentation: {e}")

    # ==================== ADVANCED OPERATIONS ====================

    async def get_update_suggestions(self, force_scan: bool = False,
                                     max_suggestions: int = 50) -> Result:
        """Get prioritized documentation update suggestions"""
        try:
            if not self.current_index:
                self.current_index = self._load_index()

            # Quick scan for changes if needed
            if force_scan:
                await self._quick_update_index()

            suggestions = []

            # Find undocumented code elements
            undocumented = self._find_undocumented_elements()
            for element in undocumented[:max_suggestions // 2]:
                priority = self._assess_priority(element)
                suggestions.append({
                    "type": "missing_documentation",
                    "element_name": element.name,
                    "element_type": element.element_type,
                    "file_path": element.file_path,
                    "priority": priority,
                    "action": "generate_from_code" if element.element_type == "class" else "add_section"
                })

            # Find unclear sections
            unclear = self._find_unclear_sections()
            for section_id in unclear[:max_suggestions // 2]:
                section = self.current_index.sections[section_id]
                suggestions.append({
                    "type": "unclear_documentation",
                    "section_id": section_id,
                    "title": section.title,
                    "file_path": section.file_path,
                    "priority": "medium",
                    "action": "update_section"
                })

            # Sort by priority
            priority_order = {"high": 0, "medium": 1, "low": 2}
            suggestions.sort(key=lambda x: priority_order.get(x["priority"], 3))

            return Result.ok({
                "suggestions": suggestions[:max_suggestions],
                "total_found": len(suggestions),
                "undocumented_elements": len(undocumented),
                "unclear_sections": len(unclear)
            })

        except Exception as e:
            logger.error(f"Error getting update suggestions: {e}")
            return Result.default_user_error(f"Error analyzing updates: {e}")

    async def auto_update_docs(self, dry_run: bool = False, max_updates: int = 5,
                               timeout: int = 30) -> Result:
        """Automatically update documentation with timeout protection"""
        try:
            # Get suggestions
            suggestions_result = await asyncio.wait_for(
                self.get_update_suggestions(max_suggestions=max_updates * 2),
                timeout=10.0
            )

            if suggestions_result.is_error():
                return suggestions_result

            suggestions = suggestions_result.get("suggestions")[:max_updates]

            if dry_run:
                return Result.ok({
                    "dry_run": True,
                    "would_update": len(suggestions),
                    "suggestions": [s["type"] + ": " + s.get("element_name", s.get("title", ""))
                                    for s in suggestions]
                })

            results = []
            start_time = time.time()

            for suggestion in suggestions:
                if time.time() - start_time > timeout:
                    break

                try:
                    if suggestion["action"] == "generate_from_code":
                        result = await asyncio.wait_for(
                            self.docs_writer(
                                action="generate_from_code",
                                source_file=suggestion["file_path"],
                                auto_generate=True
                            ), timeout=10.0
                        )
                    elif suggestion["action"] == "add_section":
                        result = await asyncio.wait_for(
                            self.docs_writer(
                                action="add_section",
                                file_path=f"{Path(suggestion['file_path']).stem}.md",
                                section_title=suggestion["element_name"],
                                source_file=suggestion["file_path"],
                                auto_generate=True
                            ), timeout=8.0
                        )
                    else:
                        continue

                    results.append({
                        "suggestion": suggestion["type"],
                        "status": "success" if result.is_ok() else "error",
                        "details": str(result.error) if result.is_error() else "completed"
                    })

                except asyncio.TimeoutError:
                    results.append({
                        "suggestion": suggestion["type"],
                        "status": "timeout",
                        "details": "Operation timed out"
                    })

                await asyncio.sleep(0.1)  # Brief pause between operations

            return Result.ok({
                "processed": len(results),
                "successful": len([r for r in results if r["status"] == "success"]),
                "results": results,
                "execution_time": f"{time.time() - start_time:.2f}s"
            })

        except Exception as e:
            logger.error(f"Error in auto_update_docs: {e}")
            return Result.default_user_error(f"Auto-update failed: {e}")

    async def initial_docs_parse(self, update_index: bool = True, force_rebuild: bool = False) -> Result:
        """Parse existing documentation and build initial index with proper saving"""
        try:
            logger.info("Starting initial documentation parse...")

            # Check if we can use existing index
            if not force_rebuild and self.index_file.exists() and not update_index:
                self.current_index = self._load_index(minimal=False)
                if self.current_index.sections or self.current_index.code_elements:
                    logger.info("Using existing index")
                    return Result.ok({
                        "total_sections": len(self.current_index.sections),
                        "total_code_elements": len(self.current_index.code_elements),
                        "linked_sections": len([s for s in self.current_index.sections.values() if s.source_refs]),
                        "completion_rate": f"{(len([s for s in self.current_index.sections.values() if s.source_refs]) / max(len(self.current_index.sections), 1) * 100):.1f}%",
                        "used_cached": True
                    })

            if update_index or force_rebuild:
                # Build comprehensive index
                logger.info("Building comprehensive index from source files...")
                self.current_index = await asyncio.get_event_loop().run_in_executor(
                    None, self._build_full_index
                )
            else:
                self.current_index = self._load_index(minimal=False)

            # Ensure we have some documentation sections
            if not self.current_index.sections:
                logger.info("No documentation sections found, scanning for markdown files...")
                # Look for markdown files in docs and project root
                search_paths = [self.docs_root]
                if self.docs_root != self.project_root:
                    search_paths.append(self.project_root)

                for search_path in search_paths:
                    for md_file in search_path.rglob('*.md'):
                        if self._should_include_file(md_file):
                            sections = self._parse_markdown_file(md_file)
                            for section in sections:
                                self.current_index.sections[section.section_id] = section

                    # Don't search too deep
                    if len(self.current_index.sections) > 0:
                        break

                logger.info(f"Found {len(self.current_index.sections)} documentation sections")

            # Link documentation sections to code elements
            linked_count = await self._link_docs_to_code()

            # Save updated index with proper error handling
            try:
                await self._save_index_async(self.current_index)
                logger.info("Index saved successfully")
            except Exception as e:
                logger.error(f"Failed to save index: {e}")
                # Continue anyway, we have the index in memory

            completion_rate = (
                    linked_count / max(len(self.current_index.sections), 1) * 100) if self.current_index.sections else 0

            return Result.ok({
                "total_sections": len(self.current_index.sections),
                "total_code_elements": len(self.current_index.code_elements),
                "linked_sections": linked_count,
                "completion_rate": f"{completion_rate:.1f}%",
                "index_file": str(self.index_file),
                "docs_root": str(self.docs_root)
            })

        except Exception as e:
            logger.error(f"Error in initial_docs_parse: {e}")
            return Result.default_user_error(f"Failed to parse docs: {e}")

    # ==================== INTERNAL HELPER METHODS ====================

    async def _search_sections(self, query: Optional[str], file_path: Optional[str],
                               tags: Optional[List[str]], max_results: int, start_time: float) -> List[DocSection]:
        """Optimized section search with early termination"""
        matching_sections = []
        search_terms = set(query.lower().split()) if query else set()

        for section in self.current_index.sections.values():
            if len(matching_sections) >= max_results:
                break

            # Timeout protection
            if time.time() - start_time > 3.0:
                break

            # Quick filters
            if file_path and file_path not in section.file_path:
                continue

            if tags and not any(tag in section.tags for tag in tags):
                continue

            if search_terms:
                search_content = f"{section.title} {section.content[:200]}".lower()
                if not any(term in search_content for term in search_terms):
                    continue

            matching_sections.append(section)

        return matching_sections

    def _format_sections(self, sections: List[DocSection], include_source_refs: bool,
                         format_type: str) -> dict:
        """Unified section formatting"""
        if format_type == "markdown":
            output = []
            for section in sections[:20]:
                output.append(f"{'#' * section.level} {section.title}")
                output.append("")
                content = section.content[:500] + ("..." if len(section.content) > 500 else "")
                output.append(content)
                if include_source_refs and section.source_refs:
                    output.append(f"\n**References:** {', '.join(section.source_refs[:3])}")
                output.append("")
            return "\n".join(output)

        elif format_type == "json":
            return [self._section_to_dict(s, include_source_refs) for s in sections[:20]]

        else:  # structured
            return {
                "sections": [self._section_to_dict(s, include_source_refs) for s in sections[:20]],
                "metadata": {
                    "total_available": len(sections),
                    "returned_sections": min(len(sections), 20),
                    "truncated": len(sections) > 20
                }
            }

    def _format_single_section(self, section: DocSection, include_source_refs: bool,
                               format_type: str) -> Result:
        """Format a single section efficiently"""
        if format_type == "markdown":
            content = f"{'#' * section.level} {section.title}\n\n{section.content}"
            if include_source_refs and section.source_refs:
                content += f"\n\n**References:** {', '.join(section.source_refs[:5])}"
            return Result.ok(content)

        elif format_type == "json":
            return Result.ok([self._section_to_dict(section, include_source_refs)])

        else:  # structured
            return Result.ok({
                "sections": [self._section_to_dict(section, include_source_refs)],
                "metadata": {"total_sections": 1, "query_type": "specific_section"}
            })

    def _section_to_dict(self, section: DocSection, include_source_refs: bool) -> dict:
        """Convert section to dictionary efficiently"""
        data = {
            "id": section.section_id,
            "title": section.title,
            "content": section.content[:500],  # Limit for performance
            "file_path": section.file_path,
            "level": section.level,
            "tags": section.tags[:3]
        }
        if include_source_refs:
            data["source_refs"] = section.source_refs[:3]
        return data

    async def _update_section(self, file_path: str, section_title: str, content: str,
                              source_file: str, auto_generate: bool) -> Result:
        """Update existing section"""
        if not file_path or not section_title:
            return Result.default_user_error("file_path and section_title required")

        section_id = f"{Path(file_path).name}#{section_title}"
        if section_id not in self.current_index.sections:
            return Result.default_user_error(f"Section not found: {section_id}")

        section = self.current_index.sections[section_id]

        if auto_generate and source_file:
            content = await self._generate_content(section_title, source_file)

        if not content:
            return Result.default_user_error("content required")

        # Update file
        success = self._update_section_in_file(section, content)
        if not success:
            return Result.default_user_error("Failed to update file")

        # Update index
        section.content = content
        section.content_hash = hashlib.md5(content.encode()).hexdigest()
        section.last_modified = datetime.now()

        asyncio.create_task(self._save_index_async(self.current_index))

        return Result.ok({"action": "section_updated", "section_id": section_id})

    async def _add_section(self, file_path: str, section_title: str, content: str,
                           source_file: str, auto_generate: bool, position: str, level: int) -> Result:
        """Add new section to file"""
        if not file_path or not section_title:
            return Result.default_user_error("file_path and section_title required")

        if auto_generate and source_file:
            content = await self._generate_content(section_title, source_file)

        if not content:
            content = f"Content for {section_title}\n\nTODO: Add documentation here."

        # Create section
        section = DocSection(
            section_id=f"{Path(file_path).name}#{section_title}",
            file_path=str(self.docs_root / file_path),
            title=section_title,
            content=content,
            level=level,
            line_start=0,
            line_end=0,
            hash_signature=hashlib.md5(content.encode()).hexdigest(),
            content_hash=hashlib.md5(content.encode()).hexdigest()
        )

        # Add to file
        success = self._add_section_to_file(file_path, section, position)
        if not success:
            return Result.default_user_error("Failed to add section to file")

        # Update index
        self.current_index.sections[section.section_id] = section
        asyncio.create_task(self._save_index_async(self.current_index))

        return Result.ok({"action": "section_added", "section_id": section.section_id})

    async def _create_file(self, file_path: str, content: str, source_file: str,
                           auto_generate: bool) -> Result:
        """Create new documentation file"""
        if not file_path:
            return Result.default_user_error("file_path required")

        full_path = self.docs_root / file_path

        if full_path.exists():
            return Result.default_user_error(f"File already exists: {file_path}")

        if auto_generate and source_file:
            content = await self._generate_file_content(source_file)

        if not content:
            title = Path(file_path).stem.replace('_', ' ').title()
            content = f"# {title}\n\nDocumentation for {title}.\n\n"

        full_path.parent.mkdir(parents=True, exist_ok=True)
        full_path.write_text(content, encoding='utf-8')

        return Result.ok({"action": "file_created", "file_path": str(full_path)})

    async def _generate_from_code(self, source_file: str, file_path: str,
                                  auto_generate: bool) -> Result:
        """Generate documentation from source code"""
        if not source_file:
            return Result.default_user_error("source_file required")

        if not Path(source_file).exists():
            return Result.default_user_error(f"Source file not found: {source_file}")

        if not file_path:
            file_path = f"{Path(source_file).stem}.md"

        content = await self._generate_file_content(source_file) if auto_generate else ""

        return await self._create_file(file_path, content, source_file, False)

    async def _generate_content(self, title: str, source_file: str) -> str:
        """Generate content using AI with timeout protection"""
        try:
            isaa = self.app.get_mod("isaa")
            agent = await isaa.get_agent("docwriter")

            if not agent:
                return f"# {title}\n\nAI agent not available. Please add content manually."

            with open(source_file, 'r', encoding='utf-8') as f:
                source_content = f.read()[:2000]  # Limit source size

            prompt = f"""Generate concise documentation for "{title}" from this code:

```python
{source_content}
```

Requirements: Clear, concise, markdown format, no section header."""

            result = await asyncio.wait_for(
                agent.a_run_llm_completion(
                    messages=[{"role": "user", "content": prompt}],
                    model_preference="fast"
                ), timeout=8.0
            )
            return result

        except Exception as e:
            logger.error(f"Error generating content: {e}")
            return f"# {title}\n\nError generating content: {e}"

    async def _generate_file_content(self, source_file: str) -> str:
        """Generate complete file documentation"""
        try:
            isaa = self.app.get_mod("isaa")
            agent = await isaa.get_agent("docwriter")

            if not agent:
                return f"# {Path(source_file).stem}\n\nAI agent not available."

            # Extract code elements
            elements = self._extract_code_elements(Path(source_file))

            with open(source_file, 'r', encoding='utf-8') as f:
                source_content = f.read()[:3000]  # Limit source size

            prompt = f"""Generate comprehensive documentation for: {source_file}

Code:
```python
{source_content}
```

Elements: {len(elements)} classes/functions found

Create complete markdown with headers, overview, and examples."""

            result = await asyncio.wait_for(
                agent.a_run_llm_completion(
                    messages=[{"role": "user", "content": prompt}],
                    model_preference="fast"
                ), timeout=12.0
            )
            return result

        except Exception as e:
            logger.error(f"Error generating file content: {e}")
            return f"# {Path(source_file).stem}\n\nError generating documentation: {e}"

    def _build_full_index(self) -> DocsIndex:
        """Build comprehensive index from scratch with better progress tracking"""
        logger.info("Building full documentation index...")

        index = DocsIndex()
        index.last_git_commit = self._get_git_commit()

        # Get target files
        target_files = self._get_target_files()
        logger.info(f"Processing {len(target_files)} files")

        code_files_processed = 0
        md_files_processed = 0

        for i, file_path in enumerate(target_files):
            try:
                if file_path.suffix == '.py':
                    elements = self._extract_code_elements(file_path)
                    for element in elements:
                        element_id = f"{element.file_path}:{element.name}"
                        if element.parent_class:
                            element_id = f"{element.file_path}:{element.parent_class}.{element.name}"
                        index.code_elements[element_id] = element
                    code_files_processed += 1

                elif file_path.suffix.lower() == '.md':
                    sections = self._parse_markdown_file(file_path)
                    for section in sections:
                        index.sections[section.section_id] = section
                    if sections:
                        md_files_processed += 1

                # Store file hash
                index.file_hashes[str(file_path)] = self._get_file_hash(file_path)

                # Progress logging
                if i % 50 == 0 and i > 0:
                    logger.info(f"Progress: {i}/{len(target_files)} files processed")

            except Exception as e:
                logger.error(f"Error processing {file_path}: {e}")

        index.last_indexed = datetime.now()
        logger.info(f"Index built: {len(index.code_elements)} elements from {code_files_processed} Python files, "
                    f"{len(index.sections)} sections from {md_files_processed} markdown files")

        return index

    async def _quick_update_index(self):
        """Quick index update for changed files only"""
        try:
            changes = await asyncio.get_event_loop().run_in_executor(
                None, self._get_git_changes
            )

            for change in changes[:20]:  # Limit changes processed
                file_path = Path(change)
                if not self._should_include_file(file_path):
                    continue

                if not file_path.exists():
                    self._remove_file_from_index(file_path)
                    continue

                new_hash = self._get_file_hash(file_path)
                old_hash = self.current_index.file_hashes.get(str(file_path))

                if new_hash != old_hash:
                    self._update_file_in_index(file_path)
                    self.current_index.file_hashes[str(file_path)] = new_hash

            self.current_index.last_indexed = datetime.now()

        except Exception as e:
            logger.error(f"Error in quick index update: {e}")

    async def _link_docs_to_code(self) -> int:
        """Link documentation sections to code elements"""
        linked_count = 0

        for section_id, section in self.current_index.sections.items():
            matches = self._find_code_matches(section)
            if matches:
                section.source_refs.extend(matches)
                section.source_refs = list(set(section.source_refs))  # Remove duplicates
                linked_count += 1

        return linked_count

    def _find_code_matches(self, section: DocSection) -> List[str]:
        """Find code elements that match a documentation section"""
        matches = []
        title_words = set(re.findall(r'\b[A-Za-z_][A-Za-z0-9_]*\b', section.title.lower()))
        content_words = set(re.findall(r'\b[A-Za-z_][A-Za-z0-9_]*\b', section.content.lower()))

        for element_id, element in self.current_index.code_elements.items():
            score = 0

            # Direct matches
            if element.name.lower() in title_words:
                score += 10
            if element.name.lower() in content_words:
                score += 5

            # File correlation
            if Path(element.file_path).stem.lower() in Path(section.file_path).stem.lower():
                score += 3

            if score >= 5:
                matches.append(element_id)

        return matches

    def _find_undocumented_elements(self) -> List[CodeElement]:
        """Find code elements without documentation"""
        undocumented = []

        for element_id, element in self.current_index.code_elements.items():
            has_docs = any(
                element_id in section.source_refs or
                element.name in section.content or
                element.name in section.title
                for section in self.current_index.sections.values()
            )

            if not has_docs:
                undocumented.append(element)

        return undocumented

    def _find_unclear_sections(self) -> List[str]:
        """Find sections with unclear content"""
        unclear = []
        unclear_indicators = ['todo', 'fixme', 'placeholder', 'coming soon', 'tbd']

        for section_id, section in self.current_index.sections.items():
            content_lower = section.content.lower()

            if (any(indicator in content_lower for indicator in unclear_indicators) or
                len(section.content.strip()) < 50):
                unclear.append(section_id)

        return unclear

    def _assess_priority(self, element: CodeElement) -> str:
        """Assess documentation priority for code element"""
        score = 0

        # Type weights
        type_scores = {'class': 5, 'function': 3, 'method': 2}
        score += type_scores.get(element.element_type, 1)

        # Complexity
        if element.signature and element.signature.count(',') > 2:
            score += 2

        # Public vs private
        if not element.name.startswith('_'):
            score += 2

        # No docstring
        if not element.docstring:
            score += 2

        return "high" if score >= 8 else "medium" if score >= 5 else "low"

    # ==================== FILE OPERATIONS ====================

    def _update_section_in_file(self, section: DocSection, new_content: str) -> bool:
        """Update section content in file"""
        try:
            file_path = Path(section.file_path)
            with open(file_path, 'r', encoding='utf-8') as f:
                lines = f.readlines()

            # Replace section content
            new_section = f"{'#' * section.level} {section.title}\n\n{new_content}\n\n"
            new_lines = new_section.split('\n')

            # Update the lines
            lines[section.line_start:section.line_end + 1] = [line + '\n' for line in new_lines]

            with open(file_path, 'w', encoding='utf-8') as f:
                f.writelines(lines)

            return True
        except Exception as e:
            logger.error(f"Error updating section in file: {e}")
            return False

    def _add_section_to_file(self, file_path: str, section: DocSection, position: str) -> bool:
        """Add section to file"""
        try:
            full_path = self.docs_root / file_path

            if not full_path.exists():
                # Create new file
                content = f"{'#' * section.level} {section.title}\n\n{section.content}\n\n"
                full_path.write_text(content, encoding='utf-8')
                return True

            # Add to existing file
            with open(full_path, 'r', encoding='utf-8') as f:
                content = f.read()

            new_section = f"\n{'#' * section.level} {section.title}\n\n{section.content}\n\n"

            if position == "top":
                content = new_section + content
            else:
                content = content + new_section

            full_path.write_text(content, encoding='utf-8')
            return True

        except Exception as e:
            logger.error(f"Error adding section to file: {e}")
            return False

    # ==================== UTILITY METHODS ====================

    def _get_target_files(self) -> List[Path]:
        """Get filtered list of target files"""
        files = []
        extensions = ['.py', '.md']

        search_dirs = [self.project_root / d for d in self.include_dirs if (self.project_root / d).exists()]
        if not search_dirs:
            search_dirs = [self.project_root]

        for search_dir in search_dirs:
            for ext in extensions:
                for file_path in search_dir.rglob(f"*{ext}"):
                    if self._should_include_file(file_path):
                        files.append(file_path)

        return list(set(files))

    def _should_include_file(self, file_path: Path) -> bool:
        """Check if file should be processed"""
        file_str = str(file_path)

        # Exclude patterns
        if any(exclude in file_str for exclude in self.exclude_dirs):
            return False

        # Include patterns
        if self.include_dirs:
            return any(file_path.is_relative_to(self.project_root / include_dir)
                       for include_dir in self.include_dirs
                       if (self.project_root / include_dir).exists())

        return True

    def _get_file_hash(self, file_path: Path) -> str:
        """Get file hash for change detection"""
        try:
            with open(file_path, 'rb') as f:
                return hashlib.md5(f.read()).hexdigest()
        except Exception:
            return ""

    def _get_git_commit(self) -> Optional[str]:
        """Get current git commit hash"""
        try:
            result = subprocess.run(
                ["git", "rev-parse", "HEAD"],
                cwd=self.project_root,
                capture_output=True,
                text=True,
                timeout=10
            )
            return result.stdout.strip() if result.returncode == 0 else None
        except Exception:
            return None

    def _get_git_changes(self) -> List[str]:
        """Get changed files from git"""
        try:
            result = subprocess.run(
                ["git", "diff", "--name-only", "HEAD~1", "HEAD"],
                cwd=self.project_root,
                capture_output=True,
                text=True,
                timeout=10
            )
            return result.stdout.strip().split('\n') if result.returncode == 0 else []
        except Exception:
            return []

    def _extract_code_elements(self, file_path: Path) -> List[CodeElement]:
        """Extract code elements from Python file"""
        elements = []

        try:
            with open(file_path, 'r', encoding='utf-8') as f:
                content = f.read()

            tree = ast.parse(content)

            for node in ast.walk(tree):
                if isinstance(node, ast.ClassDef):
                    elements.append(CodeElement(
                        name=node.name,
                        element_type='class',
                        file_path=str(file_path),
                        line_start=node.lineno,
                        line_end=getattr(node, 'end_lineno', node.lineno),
                        signature=f"class {node.name}",
                        docstring=ast.get_docstring(node),
                        hash_signature=hashlib.md5(f"class {node.name}".encode()).hexdigest()
                    ))

                    # Add methods
                    for item in node.body:
                        if isinstance(item, ast.FunctionDef):
                            elements.append(CodeElement(
                                name=item.name,
                                element_type='method',
                                file_path=str(file_path),
                                line_start=item.lineno,
                                line_end=getattr(item, 'end_lineno', item.lineno),
                                signature=f"def {item.name}",
                                docstring=ast.get_docstring(item),
                                parent_class=node.name,
                                hash_signature=hashlib.md5(f"def {item.name}".encode()).hexdigest()
                            ))

                elif isinstance(node, ast.FunctionDef):
                    elements.append(CodeElement(
                        name=node.name,
                        element_type='function',
                        file_path=str(file_path),
                        line_start=node.lineno,
                        line_end=getattr(node, 'end_lineno', node.lineno),
                        signature=f"def {node.name}",
                        docstring=ast.get_docstring(node),
                        hash_signature=hashlib.md5(f"def {node.name}".encode()).hexdigest()
                    ))

        except Exception as e:
            logger.error(f"Error extracting elements from {file_path}: {e}")

        return elements

    def _parse_markdown_file(self, file_path: Path) -> List[DocSection]:
        """Enhanced markdown file parsing with better error handling"""
        sections = []

        try:
            # Skip non-markdown files and hidden files
            if not file_path.suffix.lower() == '.md' or file_path.name.startswith('.'):
                return sections

            with open(file_path, 'r', encoding='utf-8', errors='ignore') as f:
                content = f.read()

            if not content.strip():
                return sections

            lines = content.split('\n')
            current_section = None
            section_content = []
            line_start = 0

            for i, line in enumerate(lines):
                # Look for markdown headers with improved regex
                header_match = re.match(r'^(#{1,6})\s+(.+)$', line.strip())

                if header_match:
                    # Save previous section if exists
                    if current_section:
                        section = self._create_doc_section(
                            file_path, current_section, section_content, line_start, i - 1
                        )
                        if section:  # Only add non-empty sections
                            sections.append(section)

                    # Start new section
                    level = len(header_match.group(1))
                    title = header_match.group(2).strip()

                    # Skip empty titles
                    if not title:
                        continue

                    current_section = (title, level)
                    section_content = []
                    line_start = i

                elif current_section:
                    section_content.append(line)

            # Save last section
            if current_section:
                section = self._create_doc_section(
                    file_path, current_section, section_content, line_start, len(lines) - 1
                )
                if section:
                    sections.append(section)

            if sections:
                logger.debug(f"Parsed {len(sections)} sections from {file_path}")

        except Exception as e:
            logger.error(f"Error parsing markdown {file_path}: {e}")

        return sections

    def _create_doc_section(self, file_path: Path, section_info: Tuple[str, int],
                            content_lines: List[str], line_start: int, line_end: int) -> Optional[DocSection]:
        """Create DocSection with validation"""
        try:
            title, level = section_info
            content = '\n'.join(content_lines).strip()

            # Skip sections with no meaningful content
            if len(content) < 10 and not any(c.isalnum() for c in content):
                return None

            # Extract tags and references with safer regex
            tags = []
            source_refs = []

            try:
                tags = re.findall(r'#([a-zA-Z][a-zA-Z0-9_-]*)', content)
                source_refs = re.findall(r'`([^`]+\.py:[^`]+)`', content)
            except Exception:
                pass  # If regex fails, continue with empty lists

            section_id = f"{file_path.name}#{title}"
            content_hash = hashlib.md5(content.encode()).hexdigest()
            combined_hash = hashlib.md5(f"{title}:{content}:{line_start}".encode()).hexdigest()

            return DocSection(
                section_id=section_id,
                file_path=str(file_path),
                title=title,
                content=content,
                level=level,
                line_start=line_start,
                line_end=line_end,
                source_refs=source_refs,
                tags=tags,
                hash_signature=combined_hash,
                content_hash=content_hash,
                last_modified=datetime.fromtimestamp(
                    file_path.stat().st_mtime) if file_path.exists() else datetime.now()
            )
        except Exception as e:
            logger.error(f"Error creating doc section: {e}")
            return None

    def _remove_file_from_index(self, file_path: Path):
        """Remove all references to a file from index"""
        file_str = str(file_path)

        # Remove sections
        to_remove = [k for k, v in self.current_index.sections.items() if v.file_path == file_str]
        for key in to_remove:
            del self.current_index.sections[key]

        # Remove code elements
        to_remove = [k for k, v in self.current_index.code_elements.items() if v.file_path == file_str]
        for key in to_remove:
            del self.current_index.code_elements[key]

        # Remove file hash
        if file_str in self.current_index.file_hashes:
            del self.current_index.file_hashes[file_str]

    def _update_file_in_index(self, file_path: Path):
        """Update index for a specific file"""
        # Remove old entries
        self._remove_file_from_index(file_path)

        # Add new entries
        try:
            if file_path.suffix == '.py':
                elements = self._extract_code_elements(file_path)
                for element in elements:
                    element_id = f"{element.file_path}:{element.name}"
                    if element.parent_class:
                        element_id = f"{element.file_path}:{element.parent_class}.{element.name}"
                    self.current_index.code_elements[element_id] = element

            elif file_path.suffix == '.md' and file_path.is_relative_to(self.docs_root):
                sections = self._parse_markdown_file(file_path)
                for section in sections:
                    self.current_index.sections[section.section_id] = section

        except Exception as e:
            logger.error(f"Error updating file {file_path}: {e}")


    def source_code_lookup(self,
                           element_name: Optional[str] = None,
                           file_path: Optional[str] = None,
                           element_type: Optional[str] = None,
                           max_results: int = 25,
                           return_code_block: bool = True) -> Result:
        """Look up source code elements with option to return single method code blocks"""
        try:
            if not self.current_index:
                self.current_index = self._load_index(minimal=False)

            matches = []

            for element_id, element in self.current_index.code_elements.items():
                if len(matches) >= max_results:
                    break

                # Apply filters
                if element_name and element_name.lower() not in element.name.lower():
                    continue
                if file_path and file_path not in element.file_path:
                    continue
                if element_type and element.element_type != element_type:
                    continue

                # Get code block for this specific element
                code_block = ""
                if return_code_block:
                    code_block = self._extract_single_code_block(element)

                # Find related docs
                related_docs = []
                for section_id, section in self.current_index.sections.items():
                    if len(related_docs) >= 3:  # Limit related docs
                        break
                    if (element_id in section.source_refs or
                        element.name in section.content[:200] or
                        element.name in section.title):
                        related_docs.append({
                            "section_id": section_id,
                            "title": section.title,
                            "file_path": section.file_path
                        })

                match_data = {
                    "element_id": element_id,
                    "name": element.name,
                    "type": element.element_type,
                    "signature": element.signature,
                    "file_path": element.file_path,
                    "line_start": element.line_start,
                    "line_end": element.line_end,
                    "parent_class": element.parent_class,
                    "docstring": element.docstring[:300] if element.docstring else None,
                    "related_documentation": related_docs
                }

                if return_code_block and code_block:
                    match_data["code_block"] = code_block

                matches.append(match_data)

            return Result.ok({
                "matches": matches,
                "total_matches": len(matches),
                "total_available": len(self.current_index.code_elements),
                "truncated": len(matches) >= max_results
            })

        except Exception as e:
            logger.error(f"Error in source code lookup: {e}")
            return Result.default_user_error(f"Error looking up source code: {e}")

    def _extract_single_code_block(self, element: CodeElement) -> str:
        """Extract single method/class code block from source file"""
        try:
            with open(element.file_path, 'r', encoding='utf-8') as f:
                lines = f.readlines()

            # Get lines for this element only
            start_line = max(0, element.line_start - 1)  # Convert to 0-based
            end_line = min(len(lines), element.line_end)

            element_lines = lines[start_line:end_line]

            # Clean up the code block
            if element_lines:
                # Remove common leading whitespace
                import textwrap
                code_block = textwrap.dedent(''.join(element_lines))

                # Add language hint for syntax highlighting
                return f"```python\n{code_block}```"

            return ""

        except Exception as e:
            logger.error(f"Error extracting code block for {element.name}: {e}")
            return ""

    # Add the missing method to the app registration
    def add_source_code_lookup_to_app(self):
        """Add source code lookup method to app"""
        if hasattr(self, 'app'):
            self.app.source_code_lookup = self.source_code_lookup

add_source_code_lookup_to_app()

Add source code lookup method to app

Source code in toolboxv2/utils/extras/mkdocs.py

def add_source_code_lookup_to_app(self):
    """Add source code lookup method to app"""
    if hasattr(self, 'app'):
        self.app.source_code_lookup = self.source_code_lookup

auto_update_docs(dry_run=False, max_updates=5, timeout=30) async

Automatically update documentation with timeout protection

Source code in toolboxv2/utils/extras/mkdocs.py

async def auto_update_docs(self, dry_run: bool = False, max_updates: int = 5,
                           timeout: int = 30) -> Result:
    """Automatically update documentation with timeout protection"""
    try:
        # Get suggestions
        suggestions_result = await asyncio.wait_for(
            self.get_update_suggestions(max_suggestions=max_updates * 2),
            timeout=10.0
        )

        if suggestions_result.is_error():
            return suggestions_result

        suggestions = suggestions_result.get("suggestions")[:max_updates]

        if dry_run:
            return Result.ok({
                "dry_run": True,
                "would_update": len(suggestions),
                "suggestions": [s["type"] + ": " + s.get("element_name", s.get("title", ""))
                                for s in suggestions]
            })

        results = []
        start_time = time.time()

        for suggestion in suggestions:
            if time.time() - start_time > timeout:
                break

            try:
                if suggestion["action"] == "generate_from_code":
                    result = await asyncio.wait_for(
                        self.docs_writer(
                            action="generate_from_code",
                            source_file=suggestion["file_path"],
                            auto_generate=True
                        ), timeout=10.0
                    )
                elif suggestion["action"] == "add_section":
                    result = await asyncio.wait_for(
                        self.docs_writer(
                            action="add_section",
                            file_path=f"{Path(suggestion['file_path']).stem}.md",
                            section_title=suggestion["element_name"],
                            source_file=suggestion["file_path"],
                            auto_generate=True
                        ), timeout=8.0
                    )
                else:
                    continue

                results.append({
                    "suggestion": suggestion["type"],
                    "status": "success" if result.is_ok() else "error",
                    "details": str(result.error) if result.is_error() else "completed"
                })

            except asyncio.TimeoutError:
                results.append({
                    "suggestion": suggestion["type"],
                    "status": "timeout",
                    "details": "Operation timed out"
                })

            await asyncio.sleep(0.1)  # Brief pause between operations

        return Result.ok({
            "processed": len(results),
            "successful": len([r for r in results if r["status"] == "success"]),
            "results": results,
            "execution_time": f"{time.time() - start_time:.2f}s"
        })

    except Exception as e:
        logger.error(f"Error in auto_update_docs: {e}")
        return Result.default_user_error(f"Auto-update failed: {e}")

docs_reader(query=None, section_id=None, file_path=None, tags=None, include_source_refs=True, format_type='structured', max_results=25) async

Ultra-fast unified docs reader with proper index loading

Source code in toolboxv2/utils/extras/mkdocs.py

async def docs_reader(self,
                      query: Optional[str] = None,
                      section_id: Optional[str] = None,
                      file_path: Optional[str] = None,
                      tags: Optional[List[str]] = None,
                      include_source_refs: bool = True,
                      format_type: str = "structured",
                      max_results: int = 25) -> Result:
    """Ultra-fast unified docs reader with proper index loading"""
    try:
        start_time = time.time()

        # Load index if needed - try cached first, then saved, then build
        if not self.current_index:
            # First try to load from saved file
            self.current_index = self._load_index(minimal=False)

            # If no sections found, suggest running initial_docs_parse
            if not self.current_index.sections:
                if self.index_file.exists():
                    logger.warning("Index file exists but contains no documentation sections")
                    # Try to find any markdown files
                    md_files = list(self.docs_root.rglob('*.md'))
                    if md_files:
                        logger.info(f"Found {len(md_files)} markdown files, re-parsing...")
                        # Re-parse markdown files
                        for md_file in md_files[:10]:  # Limit for quick check
                            sections = self._parse_markdown_file(md_file)
                            for section in sections:
                                self.current_index.sections[section.section_id] = section

                        if self.current_index.sections:
                            logger.info(f"Re-parsed {len(self.current_index.sections)} sections")
                            await self._save_index_async(self.current_index)

                if not self.current_index.sections:
                    return Result.default_user_error(
                        "No documentation sections found. Run initial_docs_parse() to build the documentation index, "
                        "or create some .md files in the docs directory."
                    )

        # Check cache for repeated queries
        cache_key = f"{query}:{section_id}:{file_path}:{tags}:{format_type}:{max_results}"
        if cache_key in self._search_cache:
            cache_entry = self._search_cache[cache_key]
            if time.time() - cache_entry['timestamp'] < self._cache_timeout:
                return Result.ok(cache_entry['result'])

        # Fast path for specific section
        if section_id:
            if section_id in self.current_index.sections:
                section = self.current_index.sections[section_id]
                result = self._format_single_section(section, include_source_refs, format_type)
                return result
            else:
                return Result.default_user_error(f"Section not found: {section_id}")

        # Search and format results
        matching_sections = await self._search_sections(query, file_path, tags, max_results, start_time)
        result_data = self._format_sections(matching_sections, include_source_refs, format_type)

        # Cache the result
        self._search_cache[cache_key] = {
            'result': result_data,
            'timestamp': time.time()
        }

        return Result.ok(result_data)

    except Exception as e:
        logger.error(f"Error in docs_reader: {e}")
        return Result.default_user_error(f"Error reading documentation: {e}")

docs_writer(action, file_path=None, section_title=None, content=None, source_file=None, auto_generate=False, position=None, level=2) async

Unified optimized docs writer

Source code in toolboxv2/utils/extras/mkdocs.py

async def docs_writer(self,
                      action: str,
                      file_path: Optional[str] = None,
                      section_title: Optional[str] = None,
                      content: Optional[str] = None,
                      source_file: Optional[str] = None,
                      auto_generate: bool = False,
                      position: Optional[str] = None,
                      level: int = 2) -> Result:
    """Unified optimized docs writer"""
    try:
        if not self.current_index:
            self.current_index = self._load_index(minimal=True)

        result = {"action": action, "timestamp": datetime.now().isoformat()}

        if action == "update_section":
            return await self._update_section(file_path, section_title, content,
                                              source_file, auto_generate)
        elif action == "add_section":
            return await self._add_section(file_path, section_title, content,
                                           source_file, auto_generate, position, level)
        elif action == "create_file":
            return await self._create_file(file_path, content, source_file, auto_generate)
        elif action == "generate_from_code":
            return await self._generate_from_code(source_file, file_path, auto_generate)
        else:
            return Result.default_user_error(f"Unknown action: {action}")

    except Exception as e:
        logger.error(f"Error in docs_writer: {e}")
        return Result.default_user_error(f"Error writing documentation: {e}")

get_update_suggestions(force_scan=False, max_suggestions=50) async

Get prioritized documentation update suggestions

Source code in toolboxv2/utils/extras/mkdocs.py

async def get_update_suggestions(self, force_scan: bool = False,
                                 max_suggestions: int = 50) -> Result:
    """Get prioritized documentation update suggestions"""
    try:
        if not self.current_index:
            self.current_index = self._load_index()

        # Quick scan for changes if needed
        if force_scan:
            await self._quick_update_index()

        suggestions = []

        # Find undocumented code elements
        undocumented = self._find_undocumented_elements()
        for element in undocumented[:max_suggestions // 2]:
            priority = self._assess_priority(element)
            suggestions.append({
                "type": "missing_documentation",
                "element_name": element.name,
                "element_type": element.element_type,
                "file_path": element.file_path,
                "priority": priority,
                "action": "generate_from_code" if element.element_type == "class" else "add_section"
            })

        # Find unclear sections
        unclear = self._find_unclear_sections()
        for section_id in unclear[:max_suggestions // 2]:
            section = self.current_index.sections[section_id]
            suggestions.append({
                "type": "unclear_documentation",
                "section_id": section_id,
                "title": section.title,
                "file_path": section.file_path,
                "priority": "medium",
                "action": "update_section"
            })

        # Sort by priority
        priority_order = {"high": 0, "medium": 1, "low": 2}
        suggestions.sort(key=lambda x: priority_order.get(x["priority"], 3))

        return Result.ok({
            "suggestions": suggestions[:max_suggestions],
            "total_found": len(suggestions),
            "undocumented_elements": len(undocumented),
            "unclear_sections": len(unclear)
        })

    except Exception as e:
        logger.error(f"Error getting update suggestions: {e}")
        return Result.default_user_error(f"Error analyzing updates: {e}")

initial_docs_parse(update_index=True, force_rebuild=False) async

Parse existing documentation and build initial index with proper saving

Source code in toolboxv2/utils/extras/mkdocs.py

async def initial_docs_parse(self, update_index: bool = True, force_rebuild: bool = False) -> Result:
    """Parse existing documentation and build initial index with proper saving"""
    try:
        logger.info("Starting initial documentation parse...")

        # Check if we can use existing index
        if not force_rebuild and self.index_file.exists() and not update_index:
            self.current_index = self._load_index(minimal=False)
            if self.current_index.sections or self.current_index.code_elements:
                logger.info("Using existing index")
                return Result.ok({
                    "total_sections": len(self.current_index.sections),
                    "total_code_elements": len(self.current_index.code_elements),
                    "linked_sections": len([s for s in self.current_index.sections.values() if s.source_refs]),
                    "completion_rate": f"{(len([s for s in self.current_index.sections.values() if s.source_refs]) / max(len(self.current_index.sections), 1) * 100):.1f}%",
                    "used_cached": True
                })

        if update_index or force_rebuild:
            # Build comprehensive index
            logger.info("Building comprehensive index from source files...")
            self.current_index = await asyncio.get_event_loop().run_in_executor(
                None, self._build_full_index
            )
        else:
            self.current_index = self._load_index(minimal=False)

        # Ensure we have some documentation sections
        if not self.current_index.sections:
            logger.info("No documentation sections found, scanning for markdown files...")
            # Look for markdown files in docs and project root
            search_paths = [self.docs_root]
            if self.docs_root != self.project_root:
                search_paths.append(self.project_root)

            for search_path in search_paths:
                for md_file in search_path.rglob('*.md'):
                    if self._should_include_file(md_file):
                        sections = self._parse_markdown_file(md_file)
                        for section in sections:
                            self.current_index.sections[section.section_id] = section

                # Don't search too deep
                if len(self.current_index.sections) > 0:
                    break

            logger.info(f"Found {len(self.current_index.sections)} documentation sections")

        # Link documentation sections to code elements
        linked_count = await self._link_docs_to_code()

        # Save updated index with proper error handling
        try:
            await self._save_index_async(self.current_index)
            logger.info("Index saved successfully")
        except Exception as e:
            logger.error(f"Failed to save index: {e}")
            # Continue anyway, we have the index in memory

        completion_rate = (
                linked_count / max(len(self.current_index.sections), 1) * 100) if self.current_index.sections else 0

        return Result.ok({
            "total_sections": len(self.current_index.sections),
            "total_code_elements": len(self.current_index.code_elements),
            "linked_sections": linked_count,
            "completion_rate": f"{completion_rate:.1f}%",
            "index_file": str(self.index_file),
            "docs_root": str(self.docs_root)
        })

    except Exception as e:
        logger.error(f"Error in initial_docs_parse: {e}")
        return Result.default_user_error(f"Failed to parse docs: {e}")

source_code_lookup(element_name=None, file_path=None, element_type=None, max_results=25, return_code_block=True)

Look up source code elements with option to return single method code blocks

Source code in toolboxv2/utils/extras/mkdocs.py

def source_code_lookup(self,
                       element_name: Optional[str] = None,
                       file_path: Optional[str] = None,
                       element_type: Optional[str] = None,
                       max_results: int = 25,
                       return_code_block: bool = True) -> Result:
    """Look up source code elements with option to return single method code blocks"""
    try:
        if not self.current_index:
            self.current_index = self._load_index(minimal=False)

        matches = []

        for element_id, element in self.current_index.code_elements.items():
            if len(matches) >= max_results:
                break

            # Apply filters
            if element_name and element_name.lower() not in element.name.lower():
                continue
            if file_path and file_path not in element.file_path:
                continue
            if element_type and element.element_type != element_type:
                continue

            # Get code block for this specific element
            code_block = ""
            if return_code_block:
                code_block = self._extract_single_code_block(element)

            # Find related docs
            related_docs = []
            for section_id, section in self.current_index.sections.items():
                if len(related_docs) >= 3:  # Limit related docs
                    break
                if (element_id in section.source_refs or
                    element.name in section.content[:200] or
                    element.name in section.title):
                    related_docs.append({
                        "section_id": section_id,
                        "title": section.title,
                        "file_path": section.file_path
                    })

            match_data = {
                "element_id": element_id,
                "name": element.name,
                "type": element.element_type,
                "signature": element.signature,
                "file_path": element.file_path,
                "line_start": element.line_start,
                "line_end": element.line_end,
                "parent_class": element.parent_class,
                "docstring": element.docstring[:300] if element.docstring else None,
                "related_documentation": related_docs
            }

            if return_code_block and code_block:
                match_data["code_block"] = code_block

            matches.append(match_data)

        return Result.ok({
            "matches": matches,
            "total_matches": len(matches),
            "total_available": len(self.current_index.code_elements),
            "truncated": len(matches) >= max_results
        })

    except Exception as e:
        logger.error(f"Error in source code lookup: {e}")
        return Result.default_user_error(f"Error looking up source code: {e}")

MarkdownParser

Parses markdown files and extracts sections

Source code in toolboxv2/utils/extras/mkdocs.py

class MarkdownParser:
    """Parses markdown files and extracts sections"""

    def parse_file(self, file_path: Path) -> List[DocSection]:
        """Parse markdown file into sections"""
        sections = []

        try:
            with open(file_path, 'r', encoding='utf-8') as f:
                content = f.read()

            lines = content.split('\n')
            current_section = None
            section_content = []
            line_start = 0

            for i, line in enumerate(lines):
                # Check for header
                header_match = re.match(r'^(#{1,6})\s+(.+)$', line)

                if header_match:
                    # Save previous section
                    if current_section:
                        sections.append(self._create_section(
                            file_path, current_section, section_content,
                            line_start, i - 1
                        ))

                    # Start new section
                    level = len(header_match.group(1))
                    title = header_match.group(2).strip()
                    current_section = (title, level)
                    section_content = []
                    line_start = i

                elif current_section:
                    section_content.append(line)

            # Save last section
            if current_section:
                sections.append(self._create_section(
                    file_path, current_section, section_content,
                    line_start, len(lines) - 1
                ))

        except Exception as e:
            logger.error(f"Error parsing markdown file {file_path}: {e}")

        return sections

    def _create_section(self, file_path: Path, section_info: Tuple[str, int],
                        content_lines: List[str], line_start: int, line_end: int) -> DocSection:
        """Create DocSection from parsed data"""
        title, level = section_info
        content = '\n'.join(content_lines).strip()

        # Extract source references from content
        source_refs = self._extract_source_refs(content)

        # Extract tags
        tags = self._extract_tags(content)

        section_id = f"{file_path.name}#{title}"

        return DocSection(
            section_id=section_id,
            file_path=str(file_path),
            title=title,
            content=content,
            level=level,
            line_start=line_start,
            line_end=line_end,
            source_refs=source_refs,
            tags=tags,
            hash_signature=hashlib.md5(content.encode()).hexdigest(),
            last_modified=datetime.fromtimestamp(file_path.stat().st_mtime)
        )

    def _extract_source_refs(self, content: str) -> List[str]:
        """Extract source code references from markdown content"""
        refs = []

        # Look for code references in various formats
        patterns = [
            r'`([^`]+\.py:[^`]+)`',  # `file.py:Class.method`
            r'\[([^\]]+)\]\([^)]*\.py[^)]*\)',  # [text](file.py)
            r'```python[^`]*?([a-zA-Z_][a-zA-Z0-9_]*\.[a-zA-Z_][a-zA-Z0-9_]*)[^`]*?```'  # code blocks
        ]

        for pattern in patterns:
            matches = re.findall(pattern, content)
            refs.extend(matches)

        return list(set(refs))  # Remove duplicates

    def _extract_tags(self, content: str) -> List[str]:
        """Extract tags from markdown content"""
        tags = []

        # Look for tags in various formats
        tag_patterns = [
            r'Tags?:\s*([^\n]+)',  # Tags: tag1, tag2
            r'#([a-zA-Z][a-zA-Z0-9_-]*)',  # #hashtag
        ]

        for pattern in tag_patterns:
            matches = re.findall(pattern, content, re.IGNORECASE)
            for match in matches:
                if ',' in match:
                    tags.extend([tag.strip() for tag in match.split(',')])
                else:
                    tags.append(match.strip())

        return list(set(tags))

    def parse_file_incremental(self, file_path: Path, existing_sections: Dict[str, DocSection] = None) -> Tuple[
        List[DocSection], List[str]]:
        """Parse file with section-level change detection"""
        if existing_sections is None:
            existing_sections = {}

        new_sections = []
        changed_sections = []

        try:
            with open(file_path, 'r', encoding='utf-8') as f:
                content = f.read()

            lines = content.split('\n')
            current_section = None
            section_content = []
            line_start = 0

            for i, line in enumerate(lines):
                header_match = re.match(r'^(#{1,6})\s+(.+)$', line)

                if header_match:
                    # Process previous section
                    if current_section:
                        section = self._create_section_with_hash(
                            file_path, current_section, section_content,
                            line_start, i - 1
                        )

                        # Check if section changed
                        existing_section = existing_sections.get(section.section_id)
                        if self._section_changed(section, existing_section):
                            section.change_detected = True
                            changed_sections.append(section.section_id)

                        new_sections.append(section)

                    # Start new section
                    level = len(header_match.group(1))
                    title = header_match.group(2).strip()
                    current_section = (title, level)
                    section_content = []
                    line_start = i

                elif current_section:
                    section_content.append(line)

            # Handle last section
            if current_section:
                section = self._create_section_with_hash(
                    file_path, current_section, section_content,
                    line_start, len(lines) - 1
                )

                existing_section = existing_sections.get(section.section_id)
                if self._section_changed(section, existing_section):
                    section.change_detected = True
                    changed_sections.append(section.section_id)

                new_sections.append(section)

        except Exception as e:
            logger.error(f"Error parsing markdown file {file_path}: {e}")

        return new_sections, changed_sections

    def _create_section_with_hash(self, file_path: Path, section_info: Tuple[str, int],
                                  content_lines: List[str], line_start: int, line_end: int) -> DocSection:
        """Create DocSection with precise hash tracking"""
        title, level = section_info
        content = '\n'.join(content_lines).strip()

        # Create separate hashes for different aspects
        content_hash = hashlib.md5(content.encode()).hexdigest()
        title_hash = hashlib.md5(title.encode()).hexdigest()
        combined_hash = hashlib.md5(f"{title}:{content}:{line_start}:{line_end}".encode()).hexdigest()

        source_refs = self._extract_source_refs(content)
        tags = self._extract_tags(content)
        section_id = f"{file_path.name}#{title}"

        return DocSection(
            section_id=section_id,
            file_path=str(file_path),
            title=title,
            content=content,
            level=level,
            line_start=line_start,
            line_end=line_end,
            source_refs=source_refs,
            tags=tags,
            hash_signature=combined_hash,
            content_hash=content_hash,
            last_modified=datetime.fromtimestamp(file_path.stat().st_mtime),
            change_detected=False
        )

    def _section_changed(self, new_section: DocSection, existing_section: Optional[DocSection]) -> bool:
        """Check if section actually changed"""
        if not existing_section:
            return True  # New section

        # Quick hash comparison
        if new_section.content_hash == existing_section.content_hash:
            return False

        # Title change
        if new_section.title != existing_section.title:
            return True

        # Significant content change (not just whitespace)
        if new_section.hash_signature != existing_section.hash_signature:
            return True

        return False

parse_file(file_path)

Parse markdown file into sections

Source code in toolboxv2/utils/extras/mkdocs.py

def parse_file(self, file_path: Path) -> List[DocSection]:
    """Parse markdown file into sections"""
    sections = []

    try:
        with open(file_path, 'r', encoding='utf-8') as f:
            content = f.read()

        lines = content.split('\n')
        current_section = None
        section_content = []
        line_start = 0

        for i, line in enumerate(lines):
            # Check for header
            header_match = re.match(r'^(#{1,6})\s+(.+)$', line)

            if header_match:
                # Save previous section
                if current_section:
                    sections.append(self._create_section(
                        file_path, current_section, section_content,
                        line_start, i - 1
                    ))

                # Start new section
                level = len(header_match.group(1))
                title = header_match.group(2).strip()
                current_section = (title, level)
                section_content = []
                line_start = i

            elif current_section:
                section_content.append(line)

        # Save last section
        if current_section:
            sections.append(self._create_section(
                file_path, current_section, section_content,
                line_start, len(lines) - 1
            ))

    except Exception as e:
        logger.error(f"Error parsing markdown file {file_path}: {e}")

    return sections

parse_file_incremental(file_path, existing_sections=None)

Parse file with section-level change detection

Source code in toolboxv2/utils/extras/mkdocs.py

def parse_file_incremental(self, file_path: Path, existing_sections: Dict[str, DocSection] = None) -> Tuple[
    List[DocSection], List[str]]:
    """Parse file with section-level change detection"""
    if existing_sections is None:
        existing_sections = {}

    new_sections = []
    changed_sections = []

    try:
        with open(file_path, 'r', encoding='utf-8') as f:
            content = f.read()

        lines = content.split('\n')
        current_section = None
        section_content = []
        line_start = 0

        for i, line in enumerate(lines):
            header_match = re.match(r'^(#{1,6})\s+(.+)$', line)

            if header_match:
                # Process previous section
                if current_section:
                    section = self._create_section_with_hash(
                        file_path, current_section, section_content,
                        line_start, i - 1
                    )

                    # Check if section changed
                    existing_section = existing_sections.get(section.section_id)
                    if self._section_changed(section, existing_section):
                        section.change_detected = True
                        changed_sections.append(section.section_id)

                    new_sections.append(section)

                # Start new section
                level = len(header_match.group(1))
                title = header_match.group(2).strip()
                current_section = (title, level)
                section_content = []
                line_start = i

            elif current_section:
                section_content.append(line)

        # Handle last section
        if current_section:
            section = self._create_section_with_hash(
                file_path, current_section, section_content,
                line_start, len(lines) - 1
            )

            existing_section = existing_sections.get(section.section_id)
            if self._section_changed(section, existing_section):
                section.change_detected = True
                changed_sections.append(section.section_id)

            new_sections.append(section)

    except Exception as e:
        logger.error(f"Error parsing markdown file {file_path}: {e}")

    return new_sections, changed_sections

SectionManager

Manages precise operations on documentation sections

Source code in toolboxv2/utils/extras/mkdocs.py

class SectionManager:
    """Manages precise operations on documentation sections"""

    def __init__(self, docs_root: Path):
        self.docs_root = docs_root

    def create_new_file(self, file_path: str, initial_content: str = "") -> Result:
        """Create a new markdown file"""
        try:
            full_path = self.docs_root / file_path
            full_path.parent.mkdir(parents=True, exist_ok=True)

            if full_path.exists():
                return Result.default_user_error(f"File already exists: {file_path}")

            with open(full_path, 'w', encoding='utf-8') as f:
                f.write(initial_content)

            return Result.ok({"file_path": str(full_path), "action": "created"})

        except Exception as e:
            return Result.default_user_error(f"Error creating file: {e}")

    def add_section(self, file_path: str, section: DocSection, position: Optional[str] = None) -> Result:
        """Add a new section to a markdown file"""
        try:
            full_path = self.docs_root / file_path

            if not full_path.exists():
                # Create new file
                content = f"{'#' * section.level} {section.title}\n\n{section.content}\n\n"
                with open(full_path, 'w', encoding='utf-8') as f:
                    f.write(content)
                return Result.ok({"action": "created_file_with_section"})

            # Read existing content
            with open(full_path, 'r', encoding='utf-8') as f:
                lines = f.readlines()

            # Find insertion point
            insert_index = len(lines)  # Default: append at end

            if position:
                if position == "top":
                    insert_index = 0
                elif position.startswith("after:"):
                    target_section = position[6:]
                    for i, line in enumerate(lines):
                        if line.strip().endswith(target_section):
                            # Find end of this section
                            for j in range(i + 1, len(lines)):
                                if re.match(r'^#{1,6}\s+', lines[j]):
                                    insert_index = j
                                    break
                            break

            # Insert new section
            new_content = f"{'#' * section.level} {section.title}\n\n{section.content}\n\n"
            lines.insert(insert_index, new_content)

            # Write back
            with open(full_path, 'w', encoding='utf-8') as f:
                f.writelines(lines)

            return Result.ok({"action": "section_added", "position": insert_index})

        except Exception as e:
            return Result.default_user_error(f"Error adding section: {e}")

    def update_section(self, section: DocSection, new_content: str) -> Result:
        """Update an existing section"""
        try:
            file_path = Path(section.file_path)

            with open(file_path, 'r', encoding='utf-8') as f:
                lines = f.readlines()

            # Replace content between line_start and line_end
            new_section_content = f"{'#' * section.level} {section.title}\n\n{new_content}\n\n"
            new_lines = new_section_content.split('\n')

            # Replace the section
            lines[section.line_start:section.line_end + 1] = [line + '\n' for line in new_lines]

            # Write back
            with open(file_path, 'w', encoding='utf-8') as f:
                f.writelines(lines)

            return Result.ok({"action": "section_updated"})

        except Exception as e:
            return Result.default_user_error(f"Error updating section: {e}")

    def delete_section(self, section: DocSection) -> Result:
        """Delete a section from a file"""
        try:
            file_path = Path(section.file_path)

            with open(file_path, 'r', encoding='utf-8') as f:
                lines = f.readlines()

            # Remove lines for this section
            del lines[section.line_start:section.line_end + 1]

            # Write back
            with open(file_path, 'w', encoding='utf-8') as f:
                f.writelines(lines)

            return Result.ok({"action": "section_deleted"})

        except Exception as e:
            return Result.default_user_error(f"Error deleting section: {e}")

add_section(file_path, section, position=None)

Add a new section to a markdown file

Source code in toolboxv2/utils/extras/mkdocs.py

def add_section(self, file_path: str, section: DocSection, position: Optional[str] = None) -> Result:
    """Add a new section to a markdown file"""
    try:
        full_path = self.docs_root / file_path

        if not full_path.exists():
            # Create new file
            content = f"{'#' * section.level} {section.title}\n\n{section.content}\n\n"
            with open(full_path, 'w', encoding='utf-8') as f:
                f.write(content)
            return Result.ok({"action": "created_file_with_section"})

        # Read existing content
        with open(full_path, 'r', encoding='utf-8') as f:
            lines = f.readlines()

        # Find insertion point
        insert_index = len(lines)  # Default: append at end

        if position:
            if position == "top":
                insert_index = 0
            elif position.startswith("after:"):
                target_section = position[6:]
                for i, line in enumerate(lines):
                    if line.strip().endswith(target_section):
                        # Find end of this section
                        for j in range(i + 1, len(lines)):
                            if re.match(r'^#{1,6}\s+', lines[j]):
                                insert_index = j
                                break
                        break

        # Insert new section
        new_content = f"{'#' * section.level} {section.title}\n\n{section.content}\n\n"
        lines.insert(insert_index, new_content)

        # Write back
        with open(full_path, 'w', encoding='utf-8') as f:
            f.writelines(lines)

        return Result.ok({"action": "section_added", "position": insert_index})

    except Exception as e:
        return Result.default_user_error(f"Error adding section: {e}")

create_new_file(file_path, initial_content='')

Create a new markdown file

Source code in toolboxv2/utils/extras/mkdocs.py

def create_new_file(self, file_path: str, initial_content: str = "") -> Result:
    """Create a new markdown file"""
    try:
        full_path = self.docs_root / file_path
        full_path.parent.mkdir(parents=True, exist_ok=True)

        if full_path.exists():
            return Result.default_user_error(f"File already exists: {file_path}")

        with open(full_path, 'w', encoding='utf-8') as f:
            f.write(initial_content)

        return Result.ok({"file_path": str(full_path), "action": "created"})

    except Exception as e:
        return Result.default_user_error(f"Error creating file: {e}")

delete_section(section)

Delete a section from a file

Source code in toolboxv2/utils/extras/mkdocs.py

def delete_section(self, section: DocSection) -> Result:
    """Delete a section from a file"""
    try:
        file_path = Path(section.file_path)

        with open(file_path, 'r', encoding='utf-8') as f:
            lines = f.readlines()

        # Remove lines for this section
        del lines[section.line_start:section.line_end + 1]

        # Write back
        with open(file_path, 'w', encoding='utf-8') as f:
            f.writelines(lines)

        return Result.ok({"action": "section_deleted"})

    except Exception as e:
        return Result.default_user_error(f"Error deleting section: {e}")

update_section(section, new_content)

Update an existing section

Source code in toolboxv2/utils/extras/mkdocs.py

def update_section(self, section: DocSection, new_content: str) -> Result:
    """Update an existing section"""
    try:
        file_path = Path(section.file_path)

        with open(file_path, 'r', encoding='utf-8') as f:
            lines = f.readlines()

        # Replace content between line_start and line_end
        new_section_content = f"{'#' * section.level} {section.title}\n\n{new_content}\n\n"
        new_lines = new_section_content.split('\n')

        # Replace the section
        lines[section.line_start:section.line_end + 1] = [line + '\n' for line in new_lines]

        # Write back
        with open(file_path, 'w', encoding='utf-8') as f:
            f.writelines(lines)

        return Result.ok({"action": "section_updated"})

    except Exception as e:
        return Result.default_user_error(f"Error updating section: {e}")

TOCEntry

Represents a table of contents entry

Source code in toolboxv2/utils/extras/mkdocs.py

class TOCEntry:
    """Represents a table of contents entry"""

    def __init__(self, title: str, file_path: str, level: int, line_number: int):
        self.title = title
        self.file_path = file_path
        self.level = level
        self.line_number = line_number
        self.has_implementation = False
        self.is_unclear = False
        self.source_refs: List[str] = []

add_to_app(app, include_dirs=None, exclude_dirs=None)

Add optimized markdown docs system to app

Source code in toolboxv2/utils/extras/mkdocs.py

def add_to_app(app: AppType, include_dirs: List[str] = None, exclude_dirs: List[str] = None) -> MarkdownDocsSystem:
    """Add optimized markdown docs system to app"""
    from toolboxv2 import tb_root_dir

    docs_system = MarkdownDocsSystem(
        app,
        docs_root=str(tb_root_dir.parent / "docs"),
        include_dirs=include_dirs,
        exclude_dirs=exclude_dirs
    )

    # Register core functions
    app.docs_reader = docs_system.docs_reader
    app.docs_writer = docs_system.docs_writer
    app.get_update_suggestions = docs_system.get_update_suggestions
    app.auto_update_docs = docs_system.auto_update_docs
    app.initial_docs_parse = docs_system.initial_docs_parse
    app.source_code_lookup = docs_system.source_code_lookup

    return docs_system

notification

NotificationAction dataclass

Represents an action button in a notification

Source code in toolboxv2/utils/extras/notification.py

@dataclass
class NotificationAction:
    """Represents an action button in a notification"""
    id: str
    label: str
    callback: Optional[Callable[[], Any]] = None
    is_default: bool = False

NotificationDetails dataclass

Expandable details for notifications

Source code in toolboxv2/utils/extras/notification.py

@dataclass
class NotificationDetails:
    """Expandable details for notifications"""
    title: str
    content: str
    data: Optional[Dict] = None

NotificationPosition

Position options for notifications

Source code in toolboxv2/utils/extras/notification.py

class NotificationPosition(Enum):
    """Position options for notifications"""
    TOP_LEFT = "top_left"
    TOP_CENTER = "top_center"
    TOP_RIGHT = "top_right"
    CENTER_LEFT = "center_left"
    CENTER = "center"
    CENTER_RIGHT = "center_right"
    BOTTOM_LEFT = "bottom_left"
    BOTTOM_CENTER = "bottom_center"
    BOTTOM_RIGHT = "bottom_right"

NotificationSystem

Cross-platform notification system with OS integration and tkinter fallback

Source code in toolboxv2/utils/extras/notification.py

class NotificationSystem(metaclass=Singleton):
    """
    Cross-platform notification system with OS integration and tkinter fallback
    """

    def __init__(self):
        self.platform = sys.platform.lower()
        self.fallback_to_tkinter = True
        self.sound_enabled = False
        self.default_timeout = 1500 # Add default timeout in milliseconds
        self.max_timeout = 30000
        self.default_position = NotificationPosition.TOP_RIGHT
        self._test_os_notifications()

    def _test_os_notifications(self):
        """Test if OS notifications are available"""
        try:
            if self.platform.startswith('win'):
                # Test Windows toast notifications
                try:
                    import win10toast
                    # Test if we can create a ToastNotifier without errors
                    try:
                        toaster = win10toast.ToastNotifier()
                        # Test if classAtom exists (common error)
                        if not hasattr(toaster, 'classAtom'):
                            print("⚠️  win10toast library has compatibility issues. Will use alternative methods.")
                            # Don't set fallback_to_tkinter = True, we have alternatives
                        self.fallback_to_tkinter = False
                    except AttributeError:
                        print("⚠️  win10toast library has issues. Using alternative Windows notification methods.")
                        self.fallback_to_tkinter = True
                except ImportError:
                    print("⚠️  Windows toast notifications not available. Install win10toast: pip install win10toast")
                    print("    Alternative: Will try built-in Windows notification methods.")
                    self.fallback_to_tkinter = True  # We still have alternatives
            elif self.platform.startswith('darwin'):
                # Test macOS notifications
                try:
                    result = subprocess.run(['which', 'osascript'],
                                            capture_output=True, text=True)
                    if result.returncode != 0:
                        raise FileNotFoundError
                    self.fallback_to_tkinter = False
                except:
                    print("⚠️  macOS notifications not available. osascript not found.")
                    self.fallback_to_tkinter = True

            elif self.platform.startswith('linux'):
                # Test Linux notifications
                try:
                    result = subprocess.run(['which', 'notify-send'],
                                            capture_output=True, text=True)
                    if result.returncode != 0:
                        raise FileNotFoundError
                    self.fallback_to_tkinter = False
                except:
                    print(
                        "⚠️  Linux notifications not available. Install libnotify-bin: sudo apt install libnotify-bin")
                    self.fallback_to_tkinter = True
            else:
                print("⚠️  Unknown platform. Using tkinter fallback.")
                self.fallback_to_tkinter = True

        except Exception as e:
            print(f"⚠️  OS notification test failed: {e}. Using tkinter fallback.")
            self.fallback_to_tkinter = True

    def show_notification(self,
                          title: str,
                          message: str,
                          notification_type: NotificationType = NotificationType.INFO,
                          actions: List[NotificationAction] = None,
                          details: NotificationDetails = None,
                          timeout: int = None,
                          play_sound: bool = False,
                          position: NotificationPosition = None) -> Optional[str]:
        """
        Show a notification with optional actions and details

        Args:
            title (str): Title of the notification
            message (str): Main message of the notification
            notification_type (NotificationType): Type of notification
            actions (List[NotificationAction]): List of action buttons
            details (NotificationDetails): Expandable details
            timeout (int): Timeout in milliseconds
            play_sound (bool): Whether to play a sound
            position (NotificationPosition): Position on screen

        Returns the ID of the selected action, or None if dismissed
        """
        # Handle position configuration
        if position is None:
            position = self.default_position

        if timeout is None:
            timeout = self.default_timeout
        elif timeout > self.max_timeout:
            timeout = self.max_timeout
        elif timeout < 0:
            timeout = 0

        if play_sound and self.sound_enabled:
            self._play_notification_sound(notification_type)

        if self.fallback_to_tkinter or actions or details:
            # Use tkinter for complex notifications or as fallback
            return self._show_tkinter_notification(title, message, notification_type,
                                                   actions, details, timeout, position)
        else:
            # Use OS notification for simple notifications
            return self._show_os_notification(title, message, notification_type, timeout)

    def set_default_timeout(self, timeout_ms: int):
        """Set default timeout for notifications"""
        if timeout_ms < 0:
            self.default_timeout = 0  # No timeout
        elif timeout_ms > self.max_timeout:
            self.default_timeout = self.max_timeout
        else:
            self.default_timeout = timeout_ms

    def set_max_timeout(self, max_timeout_ms: int):
        """Set maximum allowed timeout"""
        if max_timeout_ms > 0:
            self.max_timeout = max_timeout_ms

    def set_default_position(self, position: NotificationPosition):
        """Set default position for notifications"""
        self.default_position = position

    def _show_os_notification(self, title: str, message: str,
                              notification_type: NotificationType, timeout: int) -> None:
        """Show OS native notification"""

        try:
            if self.platform.startswith('win'):
                self._show_windows_notification(title, message, notification_type, timeout)
            elif self.platform.startswith('darwin'):
                self._show_macos_notification(title, message, notification_type, timeout)
            elif self.platform.startswith('linux'):
                self._show_linux_notification(title, message, notification_type, timeout)
        except Exception as e:
            print(f"⚠️  OS notification failed: {e}. Falling back to tkinter.")
            return self._show_tkinter_notification(title, message, notification_type)

    def _show_windows_notification(self, title: str, message: str,
                                               notification_type: NotificationType, timeout):
        """Alternative Windows notification using ctypes"""
        try:
            import ctypes
            from ctypes import wintypes

            # Try using Windows 10+ notification API via PowerShell
            try:
                icon_map = {
                    NotificationType.INFO: "Information",
                    NotificationType.SUCCESS: "success",
                    NotificationType.WARNING: "Warning",
                    NotificationType.ERROR: "Error",
                    NotificationType.QUESTION: "Question"
                }

                icon_type = icon_map.get(notification_type, "Information")

                # PowerShell script to show notification
                ps_script = f'''
                [Windows.UI.Notifications.ToastNotificationManager, Windows.UI.Notifications, ContentType = WindowsRuntime] | Out-Null
                [Windows.UI.Notifications.ToastNotification, Windows.UI.Notifications, ContentType = WindowsRuntime] | Out-Null
                [Windows.Data.Xml.Dom.XmlDocument, Windows.Data.Xml.Dom.XmlDocument, ContentType = WindowsRuntime] | Out-Null

                $template = @"
                <toast>
                    <visual>
                        <binding template="ToastGeneric">
                            <text>{title}</text>
                            <text>{message}</text>
                        </binding>
                    </visual>
                </toast>
                "@

                $xml = New-Object Windows.Data.Xml.Dom.XmlDocument
                $xml.LoadXml($template)
                $toast = New-Object Windows.UI.Notifications.ToastNotification $xml
                [Windows.UI.Notifications.ToastNotificationManager]::CreateToastNotifier("Python App").Show($toast)
                '''

                result = subprocess.run(['powershell', '-Command', text_save(ps_script)],
                                        capture_output=True, text=True, timeout=5)

                if result.returncode == 0:
                    return

            except Exception:
                pass

            # Fallback to simple MessageBox
            MB_ICONINFORMATION = 0x40
            MB_ICONWARNING = 0x30
            MB_ICONERROR = 0x10
            MB_ICONQUESTION = 0x20

            icon_map = {
                NotificationType.INFO: MB_ICONINFORMATION,
                NotificationType.SUCCESS: MB_ICONINFORMATION,
                NotificationType.WARNING: MB_ICONWARNING,
                NotificationType.ERROR: MB_ICONERROR,
                NotificationType.QUESTION: MB_ICONQUESTION
            }

            icon = icon_map.get(notification_type, MB_ICONINFORMATION)

            # Show MessageBox in separate thread to avoid blocking
            def show_messagebox():
                try:
                    ctypes.windll.user32.MessageBoxW(0, message, title, icon)
                except:
                    pass

            threading.Thread(target=show_messagebox, daemon=True).start()

        except Exception:
            raise Exception("All Windows notification methods failed")

    def _show_macos_notification(self, title: str, message: str,
                                 notification_type: NotificationType, timeout: int):
        """macOS notification using osascript"""
        try:
            script = f'''
                display notification "{message}" with title "{title}"
            '''
            subprocess.run(['osascript', '-e', text_save(script)], check=True)
        except Exception as e:
            raise Exception(f"macOS notification failed: {e}")

    def _show_linux_notification(self, title: str, message: str,
                                 notification_type: NotificationType, timeout: int):
        """Linux notification using notify-send"""
        try:
            urgency = "normal"
            if notification_type == NotificationType.ERROR:
                urgency = "critical"
            elif notification_type == NotificationType.WARNING:
                urgency = "normal"

            icon = self._get_linux_icon(notification_type)

            subprocess.run([
                'notify-send',
                f'--urgency={urgency}',
                f'--expire-time={timeout}',
                f'--icon={icon}',
                text_save(title),
                text_save(message)
            ], check=True)
        except Exception as e:
            raise Exception(f"Linux notification failed: {e}")

    def _show_tkinter_notification(self, title: str, message: str,
                                   notification_type: NotificationType,
                                   actions: List[NotificationAction] = None,
                                   details: NotificationDetails = None,
                                   timeout: int = 5000,
                                   position: NotificationPosition = NotificationPosition.CENTER) -> Optional[str]:
        """Modern dark-themed tkinter notification dialog"""

        # Use a queue to communicate between threads
        result_queue = queue.Queue()

        def run_notification():
            try:
                import tkinter as tk
                from tkinter import ttk

                # Create root window
                root = tk.Tk()
                root.withdraw()  # Hide the root window

                # Create notification window
                window = tk.Toplevel(root)

                # Dark theme colors
                bg_color = "#2b2b2b"
                fg_color = "#ffffff"
                accent_color = self._get_accent_color(notification_type)
                button_color = "#404040"
                button_hover = "#505050"
                border_color = "#404040"

                # Remove window decorations for custom styling
                window.overrideredirect(True)
                window.configure(bg=border_color)

                # Variables for dragging (use instance variables to avoid threading issues)
                window.drag_data = {"x": 0, "y": 0}
                window.details_visible = False
                window.result = None

                # Create main container with border
                border_frame = tk.Frame(window, bg=border_color, padx=1, pady=1)
                border_frame.pack(fill=tk.BOTH, expand=True)

                main_container = tk.Frame(border_frame, bg=bg_color)
                main_container.pack(fill=tk.BOTH, expand=True)

                # Title bar for dragging and close button
                title_bar = tk.Frame(main_container, bg=accent_color, height=25)
                title_bar.pack(fill=tk.X, side=tk.TOP)
                title_bar.pack_propagate(False)

                # Window title in title bar
                title_label = tk.Label(title_bar, text="Notification",
                                       font=("Arial", 9), bg=accent_color, fg=fg_color)
                title_label.pack(side=tk.LEFT, padx=8, pady=4)

                # Close button
                def close_window():
                    window.result = None
                    result_queue.put(window.result)
                    root.quit()
                    root.destroy()

                close_btn = tk.Label(title_bar, text="✕", font=("Arial", 10, "bold"),
                                     bg=accent_color, fg=fg_color, cursor="hand2",
                                     padx=8, pady=2)
                close_btn.pack(side=tk.RIGHT)
                close_btn.bind("<Button-1>", lambda e: close_window())
                close_btn.bind("<Enter>", lambda e: close_btn.config(bg=self._lighten_color(accent_color, -0.2)))
                close_btn.bind("<Leave>", lambda e: close_btn.config(bg=accent_color))

                # Make title bar draggable
                def start_drag(event):
                    window.drag_data["x"] = event.x
                    window.drag_data["y"] = event.y

                def on_drag(event):
                    x = window.winfo_x() + (event.x - window.drag_data["x"])
                    y = window.winfo_y() + (event.y - window.drag_data["y"])
                    window.geometry(f"+{x}+{y}")

                title_bar.bind("<Button-1>", start_drag)
                title_bar.bind("<B1-Motion>", on_drag)
                title_label.bind("<Button-1>", start_drag)
                title_label.bind("<B1-Motion>", on_drag)

                # Content frame
                content_frame = tk.Frame(main_container, bg=bg_color, padx=15, pady=12)
                content_frame.pack(fill=tk.BOTH, expand=True)

                # Header with icon and title (more compact)
                header_frame = tk.Frame(content_frame, bg=bg_color)
                header_frame.pack(fill=tk.X, pady=(0, 8))

                # Notification type icon (smaller)
                icon_label = tk.Label(header_frame, text=self._get_emoji_icon(notification_type),
                                      font=("Arial", 16), bg=bg_color, fg=accent_color)
                icon_label.pack(side=tk.LEFT, padx=(0, 8))

                # Title (smaller font)
                title_text = tk.Label(header_frame, text=title, font=("Arial", 11, "bold"),
                                      bg=bg_color, fg=fg_color, wraplength=280)
                title_text.pack(side=tk.LEFT, fill=tk.X, expand=True, anchor="w")

                # Message (more compact)
                message_label = tk.Label(content_frame, text=message, font=("Arial", 9),
                                         bg=bg_color, fg=fg_color, wraplength=320, justify=tk.LEFT)
                message_label.pack(fill=tk.X, pady=(0, 8))

                # Details section (expandable) - initially hidden
                details_frame = None
                details_text_widget = None

                if details:
                    details_container = tk.Frame(content_frame, bg=bg_color)
                    details_container.pack(fill=tk.X, pady=(0, 8))

                    def toggle_details():
                        nonlocal details_frame, details_text_widget

                        if not window.details_visible:
                            # Show details
                            if details_frame is None:
                                details_frame = tk.Frame(details_container, bg=bg_color)
                                details_frame.pack(fill=tk.X, pady=(4, 0))

                                # Create scrollable text area
                                text_frame = tk.Frame(details_frame, bg="#1e1e1e")
                                text_frame.pack(fill=tk.X, pady=(0, 0))

                                details_text_widget = tk.Text(text_frame, height=5, bg="#1e1e1e", fg=fg_color,
                                                              border=0, wrap=tk.WORD, font=("Consolas", 8),
                                                              padx=8, pady=6)
                                details_text_widget.pack(fill=tk.X)

                                detail_content = f"{details.title}\n{'-' * min(40, len(details.title))}\n{details.content}"
                                if details.data:
                                    detail_content += f"\n\nData:\n{json.dumps(details.data, indent=2)}"

                                details_text_widget.insert(tk.END, detail_content)
                                details_text_widget.config(state=tk.DISABLED)

                            details_btn.config(text="▼ Hide Details")
                            details_frame.pack(fill=tk.X, pady=(4, 0))
                            window.details_visible = True

                            # Resize window
                            window.update_idletasks()
                            new_height = window.winfo_reqheight()
                            window.geometry(f"380x{new_height}")
                        else:
                            # Hide details
                            details_btn.config(text="▶ Show Details")
                            if details_frame:
                                details_frame.pack_forget()
                            window.details_visible = False

                            # Resize window back
                            window.update_idletasks()
                            new_height = window.winfo_reqheight()
                            window.geometry(f"380x{new_height}")

                    details_btn = tk.Button(details_container, text="▶ Show Details",
                                            command=toggle_details, bg=button_color, fg=fg_color,
                                            border=0, font=("Arial", 8), relief=tk.FLAT, cursor="hand2")
                    details_btn.pack(anchor=tk.W)

                # Action buttons (more compact)
                if actions:
                    button_frame = tk.Frame(content_frame, bg=bg_color)
                    button_frame.pack(fill=tk.X, side=tk.BOTTOM, pady=(8, 0))

                    for i, action in enumerate(actions):
                        def make_callback(action_id, callback):
                            def callback_wrapper():
                                window.result = action_id
                                result_queue.put(window.result)
                                if callback:
                                    # Run callback in separate thread
                                    threading.Thread(target=callback, daemon=True).start()
                                root.quit()
                                root.destroy()

                            return callback_wrapper

                        btn_bg = accent_color if action.is_default else button_color
                        btn = tk.Button(button_frame, text=action.label,
                                        command=make_callback(action.id, action.callback),
                                        bg=btn_bg, fg=fg_color, border=0,
                                        font=("Arial", 9), relief=tk.FLAT,
                                        padx=12, pady=6, cursor="hand2")
                        btn.pack(side=tk.RIGHT, padx=(4, 0))

                        # Hover effects
                        def on_enter(e, btn=btn, color=btn_bg):
                            btn.config(bg=self._lighten_color(color))

                        def on_leave(e, btn=btn, color=btn_bg):
                            btn.config(bg=color)

                        btn.bind("<Enter>", on_enter)
                        btn.bind("<Leave>", on_leave)
                else:
                    # Default OK button
                    button_frame = tk.Frame(content_frame, bg=bg_color)
                    button_frame.pack(fill=tk.X, side=tk.BOTTOM, pady=(8, 0))

                    def ok_clicked():
                        window.result = "ok"
                        result_queue.put(window.result)
                        root.quit()
                        root.destroy()

                    ok_btn = tk.Button(button_frame, text="OK", command=ok_clicked,
                                       bg=accent_color, fg=fg_color, border=0,
                                       font=("Arial", 9), relief=tk.FLAT,
                                       padx=12, pady=6, cursor="hand2")
                    ok_btn.pack(side=tk.RIGHT)

                # Set initial window size (slimmer)
                base_height = 150
                if timeout > 5000:
                    base_height += 20  # Add space for timeout indicator

                # Center window on screen
                # Position window based on specified position
                window.update()
                screen_width = window.winfo_screenwidth()
                screen_height = window.winfo_screenheight()
                window_width = 380
                window_height = window.winfo_height()

                # Calculate position based on enum
                margin = 20  # Margin from screen edges

                if position == NotificationPosition.TOP_LEFT:
                    x, y = margin, margin
                elif position == NotificationPosition.TOP_CENTER:
                    x, y = (screen_width - window_width) // 2, margin
                elif position == NotificationPosition.TOP_RIGHT:
                    x, y = screen_width - window_width - margin, margin
                elif position == NotificationPosition.CENTER_LEFT:
                    x, y = margin, (screen_height - window_height) // 2
                elif position == NotificationPosition.CENTER:
                    x, y = (screen_width - window_width) // 2, (screen_height - window_height) // 2 - 50
                elif position == NotificationPosition.CENTER_RIGHT:
                    x, y = screen_width - window_width - margin, (screen_height - window_height) // 2
                elif position == NotificationPosition.BOTTOM_LEFT:
                    x, y = margin, screen_height - window_height - margin - 50  # Account for taskbar
                elif position == NotificationPosition.BOTTOM_CENTER:
                    x, y = (screen_width - window_width) // 2, screen_height - window_height - margin - 50
                elif position == NotificationPosition.BOTTOM_RIGHT:
                    x, y = screen_width - window_width - margin, screen_height - window_height - margin - 50
                else:
                    # Default to center
                    x, y = (screen_width - window_width) // 2, (screen_height - window_height) // 2 - 50
                window.geometry(f"{window_width}x{window_height}+{x}+{y}")
                window.update()
                # Always on top and focus
                window.attributes('-topmost', True)
                window.focus_force()

                # Auto-close after timeout (if no actions)
                if not actions:
                    # Auto-close after timeout (for all notifications if timeout > 0)
                    if timeout > 0:
                        def create_auto_close():
                            def auto_close_handler():
                                try:
                                    if root.winfo_exists():
                                        window.result = 'timeout'
                                        result_queue.put('timeout')
                                        root.quit()
                                        root.destroy()
                                except tk.TclError:
                                    pass  # Window already destroyed
                                except Exception:
                                    pass  # Handle any other errors silently

                            root.after(timeout, auto_close_handler)

                        create_auto_close()

                    # Add timeout indicator if timeout > 10 seconds
                    # Alternative: Progress bar timeout indicator (replace the text version above)
                    if timeout > 5000:
                        timeout_frame = tk.Frame(content_frame, bg=bg_color)
                        timeout_frame.pack(fill=tk.X, pady=(2, 4))

                        # Progress bar for visual timeout
                        progress_bg = tk.Frame(timeout_frame, bg="#444444", height=4)
                        progress_bg.pack(fill=tk.X, pady=(0, 2))

                        progress_bar = tk.Frame(progress_bg, bg="#666666", height=4)
                        progress_bar.place(x=0, y=0, relwidth=1.0, height=4)

                        # Timeout text
                        timeout_label = tk.Label(timeout_frame,
                                                 text=f"⏱️ Auto-closes in {timeout // 1000}s",
                                                 font=("Arial", 8), bg=bg_color, fg="#888888")
                        timeout_label.pack(anchor=tk.E)

                        def setup_progress_countdown():
                            total_time = timeout // 1000
                            remaining = [total_time]

                            def update_progress():
                                try:
                                    if remaining[0] > 0 and root and root.winfo_exists():
                                        # Update text
                                        timeout_label.config(text=f"⏱️ Auto-closes in {remaining[0]}s")

                                        # Update progress bar
                                        progress_width = remaining[0] / total_time
                                        progress_bar.place(relwidth=progress_width)

                                        remaining[0] -= 1
                                        root.after(1000, update_progress)
                                    elif root and root.winfo_exists():
                                        timeout_label.config(text="⏱️ Closing...")
                                        progress_bar.place(relwidth=0)
                                except (tk.TclError, AttributeError):
                                    pass

                            root.after(1000, update_progress)

                        setup_progress_countdown()

                # Handle escape key
                def on_escape(event):
                    close_window()

                window.bind('<Escape>', on_escape)
                window.focus_set()

                # Start the GUI main loop
                root.mainloop()

            except Exception as e:
                print(f"⚠️  Tkinter notification error: {e}")
                result_queue.put(None)

        # Run notification in the main thread if possible, otherwise in a separate thread
        if threading.current_thread() is threading.main_thread():
            run_notification()
        else:
            # If not in main thread, we need to handle this differently
            gui_thread = threading.Thread(target=run_notification, daemon=True)
            gui_thread.start()
            gui_thread.join(timeout=30)  # Don't wait forever

        # Get result from queue
        try:
            if actions:
                return result_queue.get(timeout=1)
            return None
        except queue.Empty:
            return None

    def _play_notification_sound(self, notification_type: NotificationType):
        """Play appropriate sound for notification type"""
        try:
            if notification_type == NotificationType.ERROR:
                self._play_sound(frequency=800, duration=0.5)
            elif notification_type == NotificationType.WARNING:
                self._play_sound(frequency=600, duration=0.3)
            elif notification_type == NotificationType.SUCCESS:
                self._play_sound(frequency=1000, duration=0.2)
            else:
                self._play_sound(frequency=700, duration=0.3)
        except:
            pass  # Don't let sound errors break notifications

    def _play_sound(self, frequency: int = 800, duration: float = 0.3):
        """Play notification sound"""

        def play():
            try:
                if self.platform.startswith('win'):
                    import winsound
                    winsound.Beep(frequency, int(duration * 1000))
                elif self.platform.startswith('darwin'):
                    subprocess.run(['afplay', '/System/Library/Sounds/Ping.aiff'],
                                   check=True, capture_output=True)
                elif self.platform.startswith('linux'):
                    try:
                        subprocess.run(['paplay', '--raw', '--format=s16le',
                                        '--rate=44100', '--channels=1'],
                                       input=self._generate_tone_data(frequency, duration, 44100),
                                       check=True, timeout=2)
                    except:
                        print('\a')  # Fallback to system bell
                else:
                    print('\a')
            except:
                print('\a')  # Ultimate fallback

        # Play sound in separate thread to not block UI
        threading.Thread(target=play, daemon=True).start()

    def _generate_tone_data(self, frequency: int, duration: float, sample_rate: int = 44100) -> bytes:
        """Generate raw audio data for a sine wave tone"""
        import math
        import struct

        num_samples = int(sample_rate * duration)
        tone_data = []

        for i in range(num_samples):
            t = i / sample_rate
            fade = min(1.0, t * 10, (duration - t) * 10)
            sample = int(16384 * fade * math.sin(2 * math.pi * frequency * t))
            tone_data.append(struct.pack('<h', sample))

        return b''.join(tone_data)

    def _get_emoji_icon(self, notification_type: NotificationType) -> str:
        """Get emoji icon for notification type"""
        icons = {
            NotificationType.INFO: "ℹ️",
            NotificationType.SUCCESS: "✅",
            NotificationType.WARNING: "⚠️",
            NotificationType.ERROR: "❌",
            NotificationType.QUESTION: "❓"
        }
        return icons.get(notification_type, "📢")

    def _get_accent_color(self, notification_type: NotificationType) -> str:
        """Get accent color for notification type"""
        colors = {
            NotificationType.INFO: "#3498db",
            NotificationType.SUCCESS: "#27ae60",
            NotificationType.WARNING: "#f39c12",
            NotificationType.ERROR: "#e74c3c",
            NotificationType.QUESTION: "#9b59b6"
        }
        return colors.get(notification_type, "#3498db")

    def _lighten_color(self, color: str, factor: float = 0.2) -> str:
        """Lighten or darken a hex color"""
        try:
            color = color.lstrip('#')
            rgb = tuple(int(color[i:i + 2], 16) for i in (0, 2, 4))
            if factor > 0:
                # Lighten
                rgb = tuple(min(255, int(c + (255 - c) * factor)) for c in rgb)
            else:
                # Darken
                rgb = tuple(max(0, int(c * (1 + factor))) for c in rgb)
            return f"#{rgb[0]:02x}{rgb[1]:02x}{rgb[2]:02x}"
        except:
            return color

    def _get_icon_path(self, notification_type: NotificationType) -> Optional[str]:
        """Get icon path for Windows notifications"""
        return None

    def _get_linux_icon(self, notification_type: NotificationType) -> str:
        """Get Linux system icon name"""
        icons = {
            NotificationType.INFO: "dialog-information",
            NotificationType.SUCCESS: "dialog-information",
            NotificationType.WARNING: "dialog-warning",
            NotificationType.ERROR: "dialog-error",
            NotificationType.QUESTION: "dialog-question"
        }
        return icons.get(notification_type, "dialog-information")

set_default_position(position)

Set default position for notifications

Source code in toolboxv2/utils/extras/notification.py

def set_default_position(self, position: NotificationPosition):
    """Set default position for notifications"""
    self.default_position = position

set_default_timeout(timeout_ms)

Set default timeout for notifications

Source code in toolboxv2/utils/extras/notification.py

def set_default_timeout(self, timeout_ms: int):
    """Set default timeout for notifications"""
    if timeout_ms < 0:
        self.default_timeout = 0  # No timeout
    elif timeout_ms > self.max_timeout:
        self.default_timeout = self.max_timeout
    else:
        self.default_timeout = timeout_ms

set_max_timeout(max_timeout_ms)

Set maximum allowed timeout

Source code in toolboxv2/utils/extras/notification.py

def set_max_timeout(self, max_timeout_ms: int):
    """Set maximum allowed timeout"""
    if max_timeout_ms > 0:
        self.max_timeout = max_timeout_ms

show_notification(title, message, notification_type=NotificationType.INFO, actions=None, details=None, timeout=None, play_sound=False, position=None)

Show a notification with optional actions and details

Parameters:

Name	Type	Description	Default
title	str	Title of the notification	required
message	str	Main message of the notification	required
notification_type	NotificationType	Type of notification	INFO
actions	List[NotificationAction]	List of action buttons	None
details	NotificationDetails	Expandable details	None
timeout	int	Timeout in milliseconds	None
play_sound	bool	Whether to play a sound	False
position	NotificationPosition	Position on screen	None

Returns the ID of the selected action, or None if dismissed

Source code in toolboxv2/utils/extras/notification.py

def show_notification(self,
                      title: str,
                      message: str,
                      notification_type: NotificationType = NotificationType.INFO,
                      actions: List[NotificationAction] = None,
                      details: NotificationDetails = None,
                      timeout: int = None,
                      play_sound: bool = False,
                      position: NotificationPosition = None) -> Optional[str]:
    """
    Show a notification with optional actions and details

    Args:
        title (str): Title of the notification
        message (str): Main message of the notification
        notification_type (NotificationType): Type of notification
        actions (List[NotificationAction]): List of action buttons
        details (NotificationDetails): Expandable details
        timeout (int): Timeout in milliseconds
        play_sound (bool): Whether to play a sound
        position (NotificationPosition): Position on screen

    Returns the ID of the selected action, or None if dismissed
    """
    # Handle position configuration
    if position is None:
        position = self.default_position

    if timeout is None:
        timeout = self.default_timeout
    elif timeout > self.max_timeout:
        timeout = self.max_timeout
    elif timeout < 0:
        timeout = 0

    if play_sound and self.sound_enabled:
        self._play_notification_sound(notification_type)

    if self.fallback_to_tkinter or actions or details:
        # Use tkinter for complex notifications or as fallback
        return self._show_tkinter_notification(title, message, notification_type,
                                               actions, details, timeout, position)
    else:
        # Use OS notification for simple notifications
        return self._show_os_notification(title, message, notification_type, timeout)

NotificationType

Types of notifications

Source code in toolboxv2/utils/extras/notification.py

class NotificationType(Enum):
    """Types of notifications"""
    INFO = "info"
    SUCCESS = "success"
    WARNING = "warning"
    ERROR = "error"
    QUESTION = "question"

ask_question(title, message, yes_callback=None, no_callback=None, **kwargs)

Ask a yes/no question

Source code in toolboxv2/utils/extras/notification.py

def ask_question(title: str, message: str,
                 yes_callback: Callable = None,
                 no_callback: Callable = None, **kwargs) -> Optional[str]:
    """Ask a yes/no question"""
    notifier = create_notification_system()

    actions = [
        NotificationAction("yes", "Yes", yes_callback, is_default=True),
        NotificationAction("no", "No", no_callback)
    ]

    return notifier.show_notification(
        title, message, NotificationType.QUESTION, actions=actions, **kwargs
    )

create_notification_system()

Create and return a notification system instance

Source code in toolboxv2/utils/extras/notification.py

def create_notification_system() -> NotificationSystem:
    """Create and return a notification system instance"""
    return NotificationSystem()

example_notifications()

Example notification scenarios with better timing

Source code in toolboxv2/utils/extras/notification.py

def example_notifications():
    """Example notification scenarios with better timing"""

    notifier = create_notification_system()

    # Simple notification
    print("1. Simple info notification...")
    notifier.show_notification(
        title="Welcome!",
        message="Application started successfully.",
        notification_type=NotificationType.INFO
    )

    time.sleep(2)

    # Success notification
    print("2. Success notification...")
    notifier.show_notification(
        title="Task Complete",
        message="Your file has been processed successfully.",
        notification_type=NotificationType.SUCCESS
    )

    time.sleep(2)

    # Warning with details
    print("3. Warning with expandable details...")
    details = NotificationDetails(
        title="Performance Warning",
        content="The system is running low on memory. Consider closing some applications to free up resources.",
        data={
            "memory_usage": "85%",
            "available_memory": "2.1 GB",
            "total_memory": "16 GB",
            "top_processes": ["Chrome", "Visual Studio", "Photoshop"]
        }
    )

    notifier.show_notification(
        title="System Warning",
        message="High memory usage detected.",
        notification_type=NotificationType.WARNING,
        details=details
    )

    time.sleep(2)

    # Interactive notification with actions
    print("4. Interactive notification with actions...")

    def handle_update():
        print("🔄 Update initiated!")
        time.sleep(1)
        notifier.show_notification(
            title="Update Complete",
            message="Application has been updated to version 2.1.0.",
            notification_type=NotificationType.SUCCESS
        )

    def handle_remind_later():
        print("⏰ Reminder set for later!")
        notifier.show_notification(
            title="Reminder Set",
            message="You'll be reminded about the update in 1 hour.",
            notification_type=NotificationType.INFO
        )

    actions = [
        NotificationAction("update", "Update Now", handle_update, is_default=True),
        NotificationAction("later", "Remind Later", handle_remind_later),
        NotificationAction("skip", "Skip Version", lambda: print("❌ Update skipped"))
    ]

    selected_action = notifier.show_notification(
        title="Update Available",
        message="Version 2.1.0 is ready to install with bug fixes and new features.",
        notification_type=NotificationType.QUESTION,
        actions=actions,
        details=NotificationDetails(
            title="Update Information",
            content="This update includes security patches, performance improvements, and new features.",
            data={
                "version": "2.1.0",
                "size": "25.3 MB",
                "release_date": "2024-01-15",
                "changelog": [
                    "Fixed memory leak in file processing",
                    "Added dark mode support",
                    "Improved startup time by 40%",
                    "Updated dependencies for security"
                ]
            }
        )
    )

    print(f"✅ Selected action: {selected_action}")

    print("5. Testing different positions...")

    positions_to_test = [
        (NotificationPosition.TOP_RIGHT, "Top Right"),
        (NotificationPosition.BOTTOM_LEFT, "Bottom Left"),
        (NotificationPosition.TOP_CENTER, "Top Center"),
        (NotificationPosition.CENTER_RIGHT, "Center Right")
    ]
    notifier.fallback_to_tkinter = True
    for position, pos_name in positions_to_test:
        notifier.show_notification(
            title=f"{pos_name} Notification",
            message=f"This notification appears at {pos_name.lower()}",
            notification_type=NotificationType.INFO,
            position=position,
            timeout=2000
        )
        time.sleep(0.5)

quick_error(title, message, **kwargs)

Quick error notification

Source code in toolboxv2/utils/extras/notification.py

def quick_error(title: str, message: str, **kwargs):
    """Quick error notification"""
    notifier = create_notification_system()
    notifier.show_notification(title, message, NotificationType.ERROR, **kwargs)

quick_info(title, message, **kwargs)

Quick info notification

Source code in toolboxv2/utils/extras/notification.py

def quick_info(title: str, message: str, **kwargs):
    """Quick info notification"""
    notifier = create_notification_system()
    notifier.show_notification(title, message, NotificationType.INFO, **kwargs)

quick_success(title, message, **kwargs)

Quick success notification

Source code in toolboxv2/utils/extras/notification.py

def quick_success(title: str, message: str, **kwargs):
    """Quick success notification"""
    notifier = create_notification_system()
    notifier.show_notification(title, message, NotificationType.SUCCESS, **kwargs)

quick_warning(title, message, **kwargs)

Quick warning notification

Source code in toolboxv2/utils/extras/notification.py

def quick_warning(title: str, message: str, **kwargs):
    """Quick warning notification"""
    notifier = create_notification_system()
    notifier.show_notification(title, message, NotificationType.WARNING, **kwargs)

reqbuilder

generate_requirements(folder, output_file)

Generates requirements.txt for the specified folder using pipreqs.

Source code in toolboxv2/utils/extras/reqbuilder.py

def generate_requirements(folder: str, output_file: str):
    """Generates requirements.txt for the specified folder using pipreqs."""
    print(folder, output_file, os.path.abspath(os.curdir))
    print("Not Implemented ")
    """try:
        from pipreqs.pipreqs import get_all_imports
    except ImportError:
        subprocess.run([sys.executable, "-m", "pip", "install", "pipreqs"], check=True)
    from pipreqs.pipreqs import get_all_imports
    imports = set(get_all_imports(os.path.abspath(folder)))
    imports.remove('toolboxv2') if 'toolboxv2' in imports else None
    with open(os.path.abspath(output_file), "w") as f:
        f.write("\n".join(imports))"""

run_pipeline(base_dir)

Runs the entire pipeline to generate requirements files.

Source code in toolboxv2/utils/extras/reqbuilder.py

def run_pipeline(base_dir: str):
    """Runs the entire pipeline to generate requirements files."""
    toolbox_path = os.path.join(base_dir, "toolboxv2")
    utils_path = os.path.join(toolbox_path, "utils")
    mini_req_file = os.path.join(base_dir, "requirements_mini.txt")
    extras_req_file = os.path.join(base_dir, "requirements_tests.txt")

    # Step 1: Generate minimal requirements
    print("Step 1/2: ")
    generate_requirements(utils_path, mini_req_file)

    # Step 2: Generate extended requirements
    print("Step 2/2: ")
    extras_path = os.path.join(toolbox_path, "tests")
    generate_requirements(extras_path, extras_req_file)

install_support

Complete TB Language Setup - Build executable - Setup file associations - Install VS Code extension - Install PyCharm plugin

TBSetup

Complete TB Language setup manager

Source code in toolboxv2/utils/tbx/install_support.py

class TBSetup:
    """Complete TB Language setup manager"""

    def __init__(self):
        self.root = Path(__file__).parent
        self.system = platform.system()

    def setup_all(self):
        """Run complete setup"""
        print("═" * 70)
        print("  TB Language - Complete Setup")
        print("═" * 70)
        print()

        success = True

        # Step 1: Build
        if not self.build_executable():
            print("❌ Build failed!")
            return False

        # Step 2: System integration
        if not self.setup_system_integration():
            print("⚠️  System integration failed (optional)")
            success = False

        # Step 3: VS Code extension
        if not self.setup_vscode():
            print("⚠️  VS Code extension setup failed (optional)")
            success = False

        # Step 4: PyCharm plugin
        if not self.setup_pycharm():
            print("⚠️  PyCharm plugin setup failed (optional)")
            success = False

        print()
        print("═" * 70)
        if success:
            print("  ✓ Setup Complete!")
        else:
            print("  ⚠️  Setup completed with warnings")
        print("═" * 70)
        print()
        print("Next steps:")
        print("  1. Restart PyCharm and VS Code (if open)")
        print("  2. Create a test file: test.tbx")
        print("  3. Run it: tb run test.tbx")
        print("  4. Or double-click test.tbx to run")
        print("  5. Open .tbx files in PyCharm/VS Code for syntax highlighting")
        print()

        return success

    def build_executable(self):
        """Step 1: Build TB Language"""
        print("Step 1/4: Building TB Language...")
        print("-" * 70)

        result = subprocess.run([
            sys.executable,
            str(self.root / "toolbox-exec" / "tb_lang_cli.py"),
            "build"
        ])

        if result.returncode != 0:
            return False

        print("✓ Build successful")
        print()
        return True

    def setup_system_integration(self):
        """Step 2: System integration"""
        print("Step 2/4: Setting up system integration...")
        print("-" * 70)

        result = subprocess.run([
            sys.executable,
            str(self.root / "toolbox-exec" / "tb_setup.py"),
            "install"
        ])

        print()
        return result.returncode == 0

    def setup_vscode(self):
        """Step 3: VS Code extension"""
        print("Step 3/4: Installing VS Code extension...")
        print("-" * 70)

        vscode_ext = self.root / "tb-lang-vscode"
        if not vscode_ext.exists():
            print("⚠️  VS Code extension directory not found")
            print()
            return False

        try:
            # Check if npm is available
            subprocess.run(["npm", "--version"],
                           capture_output=True, check=True)

            # Install dependencies
            print("  Installing npm dependencies...")
            subprocess.run(["npm", "install"],
                           cwd=vscode_ext,
                           capture_output=True,
                           check=True)

            # Compile TypeScript
            print("  Compiling TypeScript...")
            subprocess.run(["npm", "run", "compile"],
                           cwd=vscode_ext,
                           capture_output=True,
                           check=True)

            # Try to install to VS Code
            print("  Installing to VS Code...")
            result = subprocess.run([
                "code", "--install-extension", str(vscode_ext.resolve())
            ], capture_output=True)

            if result.returncode == 0:
                print("✓ VS Code extension installed")
                print()
                return True
            else:
                print("⚠️  Could not auto-install to VS Code")
                print(f"   Manual install: code --install-extension {vscode_ext.resolve()}")
                print()
                return False

        except FileNotFoundError as e:
            print(f"⚠️  Tool not found: {e}")
            print("   npm: https://nodejs.org/")
            print("   VS Code: https://code.visualstudio.com/")
            print()
            return False
        except subprocess.CalledProcessError as e:
            print(f"⚠️  Command failed: {e}")
            print()
            return False

    def setup_pycharm(self):
        """Step 4: PyCharm plugin"""
        print("Step 4/4: Installing PyCharm plugin...")
        print("-" * 70)

        pycharm_plugin = self.root / "tb-lang-pycharm"
        if not pycharm_plugin.exists():
            print("⚠️  PyCharm plugin directory not found")
            print("   Creating plugin structure...")
            if not self.create_pycharm_plugin():
                print()
                return False

        try:
            # Build plugin JAR
            print("  Building PyCharm plugin...")
            if not self.build_pycharm_plugin():
                print("⚠️  Plugin build failed")
                print()
                return False

            # Install to PyCharm
            print("  Installing to PyCharm...")
            if not self.install_pycharm_plugin():
                print("⚠️  Auto-install failed")
                print()
                return False

            print("✓ PyCharm plugin installed")
            print("  Please restart PyCharm to activate the plugin")
            print()
            return True

        except Exception as e:
            print(f"⚠️  Error: {e}")
            print()
            return False

    def create_pycharm_plugin(self):
        """Create PyCharm plugin structure"""
        plugin_dir = self.root / "tb-lang-pycharm"
        plugin_dir.mkdir(exist_ok=True)

        # Create directory structure
        (plugin_dir / "src" / "main" / "resources" / "fileTypes").mkdir(parents=True, exist_ok=True)
        (plugin_dir / "src" / "main" / "resources" / "META-INF").mkdir(parents=True, exist_ok=True)

        return True

    def build_pycharm_plugin(self):
        """Build PyCharm plugin JAR"""
        plugin_dir = self.root / "tb-lang-pycharm"
        build_script = plugin_dir / "build_plugin.py"

        if not build_script.exists():
            # Create build script
            build_script.write_text('''#!/usr/bin/env python3
import zipfile
from pathlib import Path

plugin_dir = Path(__file__).parent
output_jar = plugin_dir / "tb-language.jar"

with zipfile.ZipFile(output_jar, 'w', zipfile.ZIP_DEFLATED) as jar:
    # Add plugin.xml
    plugin_xml = plugin_dir / "src" / "main" / "resources" / "META-INF" / "plugin.xml"
    if plugin_xml.exists():
        jar.write(plugin_xml, "META-INF/plugin.xml")

    # Add file type definition
    file_type = plugin_dir / "src" / "main" / "resources" / "fileTypes" / "TB.xml"
    if file_type.exists():
        jar.write(file_type, "fileTypes/TB.xml")

print(f"✓ Plugin built: {output_jar}")
''')
            build_script.chmod(0o755)

        # Run build script
        result = subprocess.run([sys.executable, str(build_script)],
                                capture_output=True, text=True)

        if result.returncode == 0:
            print(f"  {result.stdout.strip()}")
            return True
        else:
            print(f"  Build error: {result.stderr}")
            return False

    def install_pycharm_plugin(self):
        """Install plugin to PyCharm"""
        plugin_jar = self.root / "tb-lang-pycharm" / "tb-language.jar"

        if not plugin_jar.exists():
            print("  Plugin JAR not found")
            return False

        # Find PyCharm config directory
        pycharm_dirs = self.find_pycharm_config_dirs()

        if not pycharm_dirs:
            print("  PyCharm installation not found")
            print(f"  Manual install: Copy {plugin_jar} to PyCharm plugins directory")
            return False

        # Install to all found PyCharm installations
        installed = False
        for config_dir in pycharm_dirs:
            plugins_dir = config_dir / "plugins"
            plugins_dir.mkdir(exist_ok=True)

            dest = plugins_dir / "tb-language.jar"
            shutil.copy(plugin_jar, dest)
            print(f"  ✓ Installed to: {dest}")
            installed = True

        return installed

    def find_pycharm_config_dirs(self):
        """Find PyCharm config directories"""
        config_dirs = []
        home = Path.home()

        if self.system == "Windows":
            # Windows: C:\Users\<user>\AppData\Roaming\JetBrains\PyCharm*
            base = home / "AppData" / "Roaming" / "JetBrains"
            if base.exists():
                config_dirs.extend(base.glob("PyCharm*"))

        elif self.system == "Linux":
            # Linux: ~/.config/JetBrains/PyCharm*
            base = home / ".config" / "JetBrains"
            if base.exists():
                config_dirs.extend(base.glob("PyCharm*"))

            # Also check old location
            old_base = home / ".PyCharm*"
            config_dirs.extend(home.glob(".PyCharm*"))

        elif self.system == "Darwin":
            # macOS: ~/Library/Application Support/JetBrains/PyCharm*
            base = home / "Library" / "Application Support" / "JetBrains"
            if base.exists():
                config_dirs.extend(base.glob("PyCharm*"))

        return [d for d in config_dirs if d.is_dir()]

build_executable()

Step 1: Build TB Language

Source code in toolboxv2/utils/tbx/install_support.py

def build_executable(self):
    """Step 1: Build TB Language"""
    print("Step 1/4: Building TB Language...")
    print("-" * 70)

    result = subprocess.run([
        sys.executable,
        str(self.root / "toolbox-exec" / "tb_lang_cli.py"),
        "build"
    ])

    if result.returncode != 0:
        return False

    print("✓ Build successful")
    print()
    return True

build_pycharm_plugin()

Build PyCharm plugin JAR

Source code in toolboxv2/utils/tbx/install_support.py

    def build_pycharm_plugin(self):
        """Build PyCharm plugin JAR"""
        plugin_dir = self.root / "tb-lang-pycharm"
        build_script = plugin_dir / "build_plugin.py"

        if not build_script.exists():
            # Create build script
            build_script.write_text('''#!/usr/bin/env python3
import zipfile
from pathlib import Path

plugin_dir = Path(__file__).parent
output_jar = plugin_dir / "tb-language.jar"

with zipfile.ZipFile(output_jar, 'w', zipfile.ZIP_DEFLATED) as jar:
    # Add plugin.xml
    plugin_xml = plugin_dir / "src" / "main" / "resources" / "META-INF" / "plugin.xml"
    if plugin_xml.exists():
        jar.write(plugin_xml, "META-INF/plugin.xml")

    # Add file type definition
    file_type = plugin_dir / "src" / "main" / "resources" / "fileTypes" / "TB.xml"
    if file_type.exists():
        jar.write(file_type, "fileTypes/TB.xml")

print(f"✓ Plugin built: {output_jar}")
''')
            build_script.chmod(0o755)

        # Run build script
        result = subprocess.run([sys.executable, str(build_script)],
                                capture_output=True, text=True)

        if result.returncode == 0:
            print(f"  {result.stdout.strip()}")
            return True
        else:
            print(f"  Build error: {result.stderr}")
            return False

create_pycharm_plugin()

Create PyCharm plugin structure

Source code in toolboxv2/utils/tbx/install_support.py

def create_pycharm_plugin(self):
    """Create PyCharm plugin structure"""
    plugin_dir = self.root / "tb-lang-pycharm"
    plugin_dir.mkdir(exist_ok=True)

    # Create directory structure
    (plugin_dir / "src" / "main" / "resources" / "fileTypes").mkdir(parents=True, exist_ok=True)
    (plugin_dir / "src" / "main" / "resources" / "META-INF").mkdir(parents=True, exist_ok=True)

    return True

find_pycharm_config_dirs()

Find PyCharm config directories

Source code in toolboxv2/utils/tbx/install_support.py

def find_pycharm_config_dirs(self):
    """Find PyCharm config directories"""
    config_dirs = []
    home = Path.home()

    if self.system == "Windows":
        # Windows: C:\Users\<user>\AppData\Roaming\JetBrains\PyCharm*
        base = home / "AppData" / "Roaming" / "JetBrains"
        if base.exists():
            config_dirs.extend(base.glob("PyCharm*"))

    elif self.system == "Linux":
        # Linux: ~/.config/JetBrains/PyCharm*
        base = home / ".config" / "JetBrains"
        if base.exists():
            config_dirs.extend(base.glob("PyCharm*"))

        # Also check old location
        old_base = home / ".PyCharm*"
        config_dirs.extend(home.glob(".PyCharm*"))

    elif self.system == "Darwin":
        # macOS: ~/Library/Application Support/JetBrains/PyCharm*
        base = home / "Library" / "Application Support" / "JetBrains"
        if base.exists():
            config_dirs.extend(base.glob("PyCharm*"))

    return [d for d in config_dirs if d.is_dir()]

install_pycharm_plugin()

Install plugin to PyCharm

Source code in toolboxv2/utils/tbx/install_support.py

def install_pycharm_plugin(self):
    """Install plugin to PyCharm"""
    plugin_jar = self.root / "tb-lang-pycharm" / "tb-language.jar"

    if not plugin_jar.exists():
        print("  Plugin JAR not found")
        return False

    # Find PyCharm config directory
    pycharm_dirs = self.find_pycharm_config_dirs()

    if not pycharm_dirs:
        print("  PyCharm installation not found")
        print(f"  Manual install: Copy {plugin_jar} to PyCharm plugins directory")
        return False

    # Install to all found PyCharm installations
    installed = False
    for config_dir in pycharm_dirs:
        plugins_dir = config_dir / "plugins"
        plugins_dir.mkdir(exist_ok=True)

        dest = plugins_dir / "tb-language.jar"
        shutil.copy(plugin_jar, dest)
        print(f"  ✓ Installed to: {dest}")
        installed = True

    return installed

setup_all()

Run complete setup

Source code in toolboxv2/utils/tbx/install_support.py

def setup_all(self):
    """Run complete setup"""
    print("═" * 70)
    print("  TB Language - Complete Setup")
    print("═" * 70)
    print()

    success = True

    # Step 1: Build
    if not self.build_executable():
        print("❌ Build failed!")
        return False

    # Step 2: System integration
    if not self.setup_system_integration():
        print("⚠️  System integration failed (optional)")
        success = False

    # Step 3: VS Code extension
    if not self.setup_vscode():
        print("⚠️  VS Code extension setup failed (optional)")
        success = False

    # Step 4: PyCharm plugin
    if not self.setup_pycharm():
        print("⚠️  PyCharm plugin setup failed (optional)")
        success = False

    print()
    print("═" * 70)
    if success:
        print("  ✓ Setup Complete!")
    else:
        print("  ⚠️  Setup completed with warnings")
    print("═" * 70)
    print()
    print("Next steps:")
    print("  1. Restart PyCharm and VS Code (if open)")
    print("  2. Create a test file: test.tbx")
    print("  3. Run it: tb run test.tbx")
    print("  4. Or double-click test.tbx to run")
    print("  5. Open .tbx files in PyCharm/VS Code for syntax highlighting")
    print()

    return success

setup_pycharm()

Step 4: PyCharm plugin

Source code in toolboxv2/utils/tbx/install_support.py

def setup_pycharm(self):
    """Step 4: PyCharm plugin"""
    print("Step 4/4: Installing PyCharm plugin...")
    print("-" * 70)

    pycharm_plugin = self.root / "tb-lang-pycharm"
    if not pycharm_plugin.exists():
        print("⚠️  PyCharm plugin directory not found")
        print("   Creating plugin structure...")
        if not self.create_pycharm_plugin():
            print()
            return False

    try:
        # Build plugin JAR
        print("  Building PyCharm plugin...")
        if not self.build_pycharm_plugin():
            print("⚠️  Plugin build failed")
            print()
            return False

        # Install to PyCharm
        print("  Installing to PyCharm...")
        if not self.install_pycharm_plugin():
            print("⚠️  Auto-install failed")
            print()
            return False

        print("✓ PyCharm plugin installed")
        print("  Please restart PyCharm to activate the plugin")
        print()
        return True

    except Exception as e:
        print(f"⚠️  Error: {e}")
        print()
        return False

setup_system_integration()

Step 2: System integration

Source code in toolboxv2/utils/tbx/install_support.py

def setup_system_integration(self):
    """Step 2: System integration"""
    print("Step 2/4: Setting up system integration...")
    print("-" * 70)

    result = subprocess.run([
        sys.executable,
        str(self.root / "toolbox-exec" / "tb_setup.py"),
        "install"
    ])

    print()
    return result.returncode == 0

setup_vscode()

Step 3: VS Code extension

Source code in toolboxv2/utils/tbx/install_support.py

def setup_vscode(self):
    """Step 3: VS Code extension"""
    print("Step 3/4: Installing VS Code extension...")
    print("-" * 70)

    vscode_ext = self.root / "tb-lang-vscode"
    if not vscode_ext.exists():
        print("⚠️  VS Code extension directory not found")
        print()
        return False

    try:
        # Check if npm is available
        subprocess.run(["npm", "--version"],
                       capture_output=True, check=True)

        # Install dependencies
        print("  Installing npm dependencies...")
        subprocess.run(["npm", "install"],
                       cwd=vscode_ext,
                       capture_output=True,
                       check=True)

        # Compile TypeScript
        print("  Compiling TypeScript...")
        subprocess.run(["npm", "run", "compile"],
                       cwd=vscode_ext,
                       capture_output=True,
                       check=True)

        # Try to install to VS Code
        print("  Installing to VS Code...")
        result = subprocess.run([
            "code", "--install-extension", str(vscode_ext.resolve())
        ], capture_output=True)

        if result.returncode == 0:
            print("✓ VS Code extension installed")
            print()
            return True
        else:
            print("⚠️  Could not auto-install to VS Code")
            print(f"   Manual install: code --install-extension {vscode_ext.resolve()}")
            print()
            return False

    except FileNotFoundError as e:
        print(f"⚠️  Tool not found: {e}")
        print("   npm: https://nodejs.org/")
        print("   VS Code: https://code.visualstudio.com/")
        print()
        return False
    except subprocess.CalledProcessError as e:
        print(f"⚠️  Command failed: {e}")
        print()
        return False

main()

Main entry point

Source code in toolboxv2/utils/tbx/install_support.py

def main():
    """Main entry point"""
    import argparse

    parser = argparse.ArgumentParser(
        description="TB Language Complete Setup"
    )
    parser.add_argument('--skip-build', action='store_true',
                        help='Skip building the executable')
    parser.add_argument('--skip-system', action='store_true',
                        help='Skip system integration')
    parser.add_argument('--skip-vscode', action='store_true',
                        help='Skip VS Code extension')
    parser.add_argument('--skip-pycharm', action='store_true',
                        help='Skip PyCharm plugin')
    parser.add_argument('--pycharm-only', action='store_true',
                        help='Only setup PyCharm plugin')

    args = parser.parse_args()

    setup = TBSetup()

    if args.pycharm_only:
        success = setup.setup_pycharm()
    else:
        # Full setup with skip options
        success = True

        if not args.skip_build:
            success = setup.build_executable() and success

        if not args.skip_system:
            setup.setup_system_integration()

        if not args.skip_vscode:
            setup.setup_vscode()

        if not args.skip_pycharm:
            setup.setup_pycharm()

    sys.exit(0 if success else 1)

proxy

ProxyUtil

Source code in toolboxv2/utils/proxy/prox_util.py

class ProxyUtil:
    def __init__(self, *args, **kwargs):
        """
        Standard constructor used for arguments pass
        Do not override. Use __ainit__ instead
        """
        self.__storedargs = args, kwargs
        self.async_initialized = False

    async def __initobj(self):
        """Crutch used for __await__ after spawning"""
        # assert not self.async_initialized
        self.async_initialized = True
        # pass the parameters to __ainit__ that passed to __init__
        await self.__ainit__(*self.__storedargs[0], **self.__storedargs[1])
        return self

    def __await__(self):
        return self.__initobj().__await__()

    async def __ainit__(self, class_instance: Any, host='0.0.0.0', port=6587, timeout=6,
                        app: (App or AppType) | None = None,
                        remote_functions=None, peer=False, name='ProxyApp-client', do_connect=True, unix_socket=False,
                        test_override=False):
        self.class_instance = class_instance
        self.client = None
        self.test_override = test_override
        self.port = port
        self.host = host
        self.timeout = timeout
        if app is None:
            app = get_app("ProxyUtil")
        self.app = app
        self._name = name
        self.unix_socket = unix_socket
        if remote_functions is None:
            remote_functions = ["run_any", "a_run_any", "remove_mod", "save_load", "exit_main", "show_console", "hide_console",
                                "rrun_flow",
                                "get_autocompletion_dict",
                                "exit_main", "watch_mod"]
        self.remote_functions = remote_functions

        from toolboxv2.mods.SocketManager import SocketType
        self.connection_type = SocketType.client
        if peer:
            self.connection_type = SocketType.peer
        if do_connect:
            await self.connect()

    async def connect(self):
        client_result = await self.app.a_run_local(SOCKETMANAGER.CREATE_SOCKET,
                                           get_results=True,
                                           name=self._name,
                                           host=self.host,
                                           port=self.port,
                                           type_id=self.connection_type,
                                           max_connections=-1,
                                           return_full_object=True,
                                           test_override=self.test_override,
                                           unix_file=self.unix_socket)

        if client_result.is_error():
            raise Exception(f"Client {self._name} error: {client_result.print(False)}")
        if not client_result.is_data():
            raise Exception(f"Client {self._name} error: {client_result.print(False)}")
        # 'socket': socket,
        # 'receiver_socket': r_socket,
        # 'host': host,
        # 'port': port,
        # 'p2p-port': endpoint_port,
        # 'sender': send,
        # 'receiver_queue': receiver_queue,
        # 'connection_error': connection_error,
        # 'receiver_thread': s_thread,
        # 'keepalive_thread': keep_alive_thread,
        # 'running_dict': running_dict,
        # 'client_to_receiver_thread': to_receive,
        # 'client_receiver_threads': threeds,
        result = await client_result.aget()
        if result is None or result.get('connection_error') != 0:
            raise Exception(f"Client {self._name} error: {client_result.print(False)}")
        self.client = Result.ok(result)

    async def disconnect(self):
        time.sleep(1)
        close = self.client.get("close")
        await close()
        self.client = None

    async def reconnect(self):
        if self.client is not None:
            await self.disconnect()
        await self.connect()

    async def verify(self, message=b"verify"):
        await asyncio.sleep(1)
        # self.client.get('sender')({'keepalive': 0})
        await self.client.get('sender')(message)

    def __getattr__(self, name):

        # print(f"ProxyApp: {name}, {self.client is None}")
        if name == "on_exit":
            return self.disconnect
        if name == "rc":
            return self.reconnect

        if name == "r":
            try:
                return self.client.get('receiver_queue').get(timeout=self.timeout)
            except:
                return "No data"

        app_attr = getattr(self.class_instance, name)

        async def method(*args, **kwargs):
            # if name == 'run_any':
            #     print("method", name, kwargs.get('get_results', False), args[0])
            if self.client is None:
                await self.reconnect()
            if kwargs.get('spec', '-') == 'app':
                if asyncio.iscoroutinefunction(app_attr):
                    return await app_attr(*args, **kwargs)
                return app_attr(*args, **kwargs)
            try:
                if name in self.remote_functions:
                    if (name == 'run_any' or name == 'a_run_any') and not kwargs.get('get_results', False):
                        if asyncio.iscoroutinefunction(app_attr):
                            return await app_attr(*args, **kwargs)
                        return app_attr(*args, **kwargs)
                    if (name == 'run_any' or name == 'a_run_any') and kwargs.get('get_results', False):
                        if isinstance(args[0], Enum):
                            args = (args[0].__class__.NAME.value, args[0].value), args[1:]
                    self.app.sprint(f"Calling method {name}, {args=}, {kwargs}=")
                    await self.client.get('sender')({'name': name, 'args': args, 'kwargs': kwargs})
                    while Spinner("Waiting for result"):
                        try:
                            data = self.client.get('receiver_queue').get(timeout=self.timeout)
                            if isinstance(data, dict) and 'identifier' in data:
                                del data["identifier"]
                            if 'error' in data and 'origin' in data and 'result' in data and 'info' in data:
                                data = ApiResult(**data).as_result()
                            return data
                        except:
                            print("No data look later with class_instance.r")
                            return Result.default_internal_error("No data received from Demon."
                                                                 " uns class_instance.r to get data later")
            except:
                if self.client.get('socket') is None:
                    self.client = None
            return app_attr(*args, **kwargs)

        if callable(app_attr) and name in self.remote_functions and self.client is not None:
            return method
        return app_attr

__init__(*args, **kwargs)

Standard constructor used for arguments pass Do not override. Use ainit instead

Source code in toolboxv2/utils/proxy/prox_util.py

def __init__(self, *args, **kwargs):
    """
    Standard constructor used for arguments pass
    Do not override. Use __ainit__ instead
    """
    self.__storedargs = args, kwargs
    self.async_initialized = False

__initobj() async

Crutch used for await after spawning

Source code in toolboxv2/utils/proxy/prox_util.py

async def __initobj(self):
    """Crutch used for __await__ after spawning"""
    # assert not self.async_initialized
    self.async_initialized = True
    # pass the parameters to __ainit__ that passed to __init__
    await self.__ainit__(*self.__storedargs[0], **self.__storedargs[1])
    return self

prox_util

ProxyUtil

Source code in toolboxv2/utils/proxy/prox_util.py

class ProxyUtil:
    def __init__(self, *args, **kwargs):
        """
        Standard constructor used for arguments pass
        Do not override. Use __ainit__ instead
        """
        self.__storedargs = args, kwargs
        self.async_initialized = False

    async def __initobj(self):
        """Crutch used for __await__ after spawning"""
        # assert not self.async_initialized
        self.async_initialized = True
        # pass the parameters to __ainit__ that passed to __init__
        await self.__ainit__(*self.__storedargs[0], **self.__storedargs[1])
        return self

    def __await__(self):
        return self.__initobj().__await__()

    async def __ainit__(self, class_instance: Any, host='0.0.0.0', port=6587, timeout=6,
                        app: (App or AppType) | None = None,
                        remote_functions=None, peer=False, name='ProxyApp-client', do_connect=True, unix_socket=False,
                        test_override=False):
        self.class_instance = class_instance
        self.client = None
        self.test_override = test_override
        self.port = port
        self.host = host
        self.timeout = timeout
        if app is None:
            app = get_app("ProxyUtil")
        self.app = app
        self._name = name
        self.unix_socket = unix_socket
        if remote_functions is None:
            remote_functions = ["run_any", "a_run_any", "remove_mod", "save_load", "exit_main", "show_console", "hide_console",
                                "rrun_flow",
                                "get_autocompletion_dict",
                                "exit_main", "watch_mod"]
        self.remote_functions = remote_functions

        from toolboxv2.mods.SocketManager import SocketType
        self.connection_type = SocketType.client
        if peer:
            self.connection_type = SocketType.peer
        if do_connect:
            await self.connect()

    async def connect(self):
        client_result = await self.app.a_run_local(SOCKETMANAGER.CREATE_SOCKET,
                                           get_results=True,
                                           name=self._name,
                                           host=self.host,
                                           port=self.port,
                                           type_id=self.connection_type,
                                           max_connections=-1,
                                           return_full_object=True,
                                           test_override=self.test_override,
                                           unix_file=self.unix_socket)

        if client_result.is_error():
            raise Exception(f"Client {self._name} error: {client_result.print(False)}")
        if not client_result.is_data():
            raise Exception(f"Client {self._name} error: {client_result.print(False)}")
        # 'socket': socket,
        # 'receiver_socket': r_socket,
        # 'host': host,
        # 'port': port,
        # 'p2p-port': endpoint_port,
        # 'sender': send,
        # 'receiver_queue': receiver_queue,
        # 'connection_error': connection_error,
        # 'receiver_thread': s_thread,
        # 'keepalive_thread': keep_alive_thread,
        # 'running_dict': running_dict,
        # 'client_to_receiver_thread': to_receive,
        # 'client_receiver_threads': threeds,
        result = await client_result.aget()
        if result is None or result.get('connection_error') != 0:
            raise Exception(f"Client {self._name} error: {client_result.print(False)}")
        self.client = Result.ok(result)

    async def disconnect(self):
        time.sleep(1)
        close = self.client.get("close")
        await close()
        self.client = None

    async def reconnect(self):
        if self.client is not None:
            await self.disconnect()
        await self.connect()

    async def verify(self, message=b"verify"):
        await asyncio.sleep(1)
        # self.client.get('sender')({'keepalive': 0})
        await self.client.get('sender')(message)

    def __getattr__(self, name):

        # print(f"ProxyApp: {name}, {self.client is None}")
        if name == "on_exit":
            return self.disconnect
        if name == "rc":
            return self.reconnect

        if name == "r":
            try:
                return self.client.get('receiver_queue').get(timeout=self.timeout)
            except:
                return "No data"

        app_attr = getattr(self.class_instance, name)

        async def method(*args, **kwargs):
            # if name == 'run_any':
            #     print("method", name, kwargs.get('get_results', False), args[0])
            if self.client is None:
                await self.reconnect()
            if kwargs.get('spec', '-') == 'app':
                if asyncio.iscoroutinefunction(app_attr):
                    return await app_attr(*args, **kwargs)
                return app_attr(*args, **kwargs)
            try:
                if name in self.remote_functions:
                    if (name == 'run_any' or name == 'a_run_any') and not kwargs.get('get_results', False):
                        if asyncio.iscoroutinefunction(app_attr):
                            return await app_attr(*args, **kwargs)
                        return app_attr(*args, **kwargs)
                    if (name == 'run_any' or name == 'a_run_any') and kwargs.get('get_results', False):
                        if isinstance(args[0], Enum):
                            args = (args[0].__class__.NAME.value, args[0].value), args[1:]
                    self.app.sprint(f"Calling method {name}, {args=}, {kwargs}=")
                    await self.client.get('sender')({'name': name, 'args': args, 'kwargs': kwargs})
                    while Spinner("Waiting for result"):
                        try:
                            data = self.client.get('receiver_queue').get(timeout=self.timeout)
                            if isinstance(data, dict) and 'identifier' in data:
                                del data["identifier"]
                            if 'error' in data and 'origin' in data and 'result' in data and 'info' in data:
                                data = ApiResult(**data).as_result()
                            return data
                        except:
                            print("No data look later with class_instance.r")
                            return Result.default_internal_error("No data received from Demon."
                                                                 " uns class_instance.r to get data later")
            except:
                if self.client.get('socket') is None:
                    self.client = None
            return app_attr(*args, **kwargs)

        if callable(app_attr) and name in self.remote_functions and self.client is not None:
            return method
        return app_attr

__init__(*args, **kwargs)

Standard constructor used for arguments pass Do not override. Use ainit instead

Source code in toolboxv2/utils/proxy/prox_util.py

def __init__(self, *args, **kwargs):
    """
    Standard constructor used for arguments pass
    Do not override. Use __ainit__ instead
    """
    self.__storedargs = args, kwargs
    self.async_initialized = False

__initobj() async

Crutch used for await after spawning

Source code in toolboxv2/utils/proxy/prox_util.py

async def __initobj(self):
    """Crutch used for __await__ after spawning"""
    # assert not self.async_initialized
    self.async_initialized = True
    # pass the parameters to __ainit__ that passed to __init__
    await self.__ainit__(*self.__storedargs[0], **self.__storedargs[1])
    return self

security

Code

Source code in toolboxv2/utils/security/cryp.py

class Code:

    @staticmethod
    def DK():
        return DEVICE_KEY

    @staticmethod
    def generate_random_string(length: int) -> str:
        """
        Generiert eine zufällige Zeichenkette der angegebenen Länge.

        Args:
            length (int): Die Länge der zu generierenden Zeichenkette.

        Returns:
            str: Die generierte Zeichenkette.
        """
        return secrets.token_urlsafe(length)

    def decode_code(self, encrypted_data, key=None):

        if not isinstance(encrypted_data, str):
            encrypted_data = str(encrypted_data)

        if key is None:
            key = DEVICE_KEY()

        return self.decrypt_symmetric(encrypted_data, key)

    def encode_code(self, data, key=None):

        if not isinstance(data, str):
            data = str(data)

        if key is None:
            key = DEVICE_KEY()

        return self.encrypt_symmetric(data, key)

    @staticmethod
    def generate_seed() -> int:
        """
        Erzeugt eine zufällige Zahl als Seed.

        Returns:
            int: Eine zufällige Zahl.
        """
        return random.randint(2 ** 32 - 1, 2 ** 64 - 1)

    @staticmethod
    def one_way_hash(text: str, salt: str = '', pepper: str = '') -> str:
        """
        Erzeugt einen Hash eines gegebenen Textes mit Salt, Pepper und optional einem Seed.

        Args:
            text (str): Der zu hashende Text.
            salt (str): Der Salt-Wert.
            pepper (str): Der Pepper-Wert.
            seed (int, optional): Ein optionaler Seed-Wert. Standardmäßig None.

        Returns:
            str: Der resultierende Hash-Wert.
        """
        return hashlib.sha256((salt + text + pepper).encode()).hexdigest()

    @staticmethod
    def generate_symmetric_key(as_str=True) -> str or bytes:
        """
        Generiert einen Schlüssel für die symmetrische Verschlüsselung.

        Returns:
            str: Der generierte Schlüssel.
        """
        key = Fernet.generate_key()
        if as_str:
            key = key.decode()
        return key

    @staticmethod
    def encrypt_symmetric(text: str or bytes, key: str) -> str:
        """
        Verschlüsselt einen Text mit einem gegebenen symmetrischen Schlüssel.

        Args:
            text (str): Der zu verschlüsselnde Text.
            key (str): Der symmetrische Schlüssel.

        Returns:
            str: Der verschlüsselte Text.
        """
        if isinstance(text, str):
            text = text.encode()

        try:
            fernet = Fernet(key.encode())
            return fernet.encrypt(text).decode()
        except Exception as e:
            get_logger().error(f"Error encrypt_symmetric #{str(e)}#")
            return "Error encrypt"

    @staticmethod
    def decrypt_symmetric(encrypted_text: str, key: str, to_str=True, mute=False) -> str or bytes:
        """
        Entschlüsselt einen Text mit einem gegebenen symmetrischen Schlüssel.

        Args:
            encrypted_text (str): Der zu entschlüsselnde Text.
            key (str): Der symmetrische Schlüssel.
            to_str (bool): default true returns str if false returns bytes
        Returns:
            str: Der entschlüsselte Text.
        """

        if isinstance(key, str):
            key = key.encode()

        #try:
        fernet = Fernet(key)
        text_b = fernet.decrypt(encrypted_text)
        if not to_str:
            return text_b
        return text_b.decode()
        # except Exception as e:
        #     get_logger().error(f"Error decrypt_symmetric {e}")
        #     if not mute:
        #         raise e
        #     if not to_str:
        #         return f"Error decoding".encode()
        #     return f"Error decoding"

    @staticmethod
    def generate_asymmetric_keys() -> (str, str):
        """
        Generiert ein Paar von öffentlichen und privaten Schlüsseln für die asymmetrische Verschlüsselung.

        Args:
            seed (int, optional): Ein optionaler Seed-Wert. Standardmäßig None.

        Returns:
            (str, str): Ein Tupel aus öffentlichem und privatem Schlüssel.
        """
        private_key = rsa.generate_private_key(
            public_exponent=65537,
            key_size=2048 * 3,
        )
        public_key = private_key.public_key()

        # Serialisieren der Schlüssel
        pem_private_key = private_key.private_bytes(
            encoding=serialization.Encoding.PEM,
            format=serialization.PrivateFormat.PKCS8,
            encryption_algorithm=serialization.NoEncryption()
        ).decode()

        pem_public_key = public_key.public_bytes(
            encoding=serialization.Encoding.PEM,
            format=serialization.PublicFormat.SubjectPublicKeyInfo
        ).decode()

        return pem_public_key, pem_private_key

    @staticmethod
    def save_keys_to_files(public_key: str, private_key: str, directory: str = "keys") -> None:
        """
        Speichert die generierten Schlüssel in separate Dateien.
        Der private Schlüssel wird mit dem Device Key verschlüsselt.

        Args:
            public_key (str): Der öffentliche Schlüssel im PEM-Format
            private_key (str): Der private Schlüssel im PEM-Format
            directory (str): Das Verzeichnis, in dem die Schlüssel gespeichert werden sollen
        """
        # Erstelle das Verzeichnis, falls es nicht existiert
        os.makedirs(directory, exist_ok=True)

        # Hole den Device Key
        device_key = DEVICE_KEY()

        # Verschlüssele den privaten Schlüssel mit dem Device Key
        encrypted_private_key = Code.encrypt_symmetric(private_key, device_key)

        # Speichere den öffentlichen Schlüssel
        public_key_path = os.path.join(directory, "public_key.pem")
        with open(public_key_path, "w") as f:
            f.write(public_key)

        # Speichere den verschlüsselten privaten Schlüssel
        private_key_path = os.path.join(directory, "private_key.pem")
        with open(private_key_path, "w") as f:
            f.write(encrypted_private_key)

        print("Saved keys in ", public_key_path)

    @staticmethod
    def load_keys_from_files(directory: str = "keys") -> (str, str):
        """
        Lädt die Schlüssel aus den Dateien.
        Der private Schlüssel wird mit dem Device Key entschlüsselt.

        Args:
            directory (str): Das Verzeichnis, aus dem die Schlüssel geladen werden sollen

        Returns:
            (str, str): Ein Tupel aus öffentlichem und privatem Schlüssel

        Raises:
            FileNotFoundError: Wenn die Schlüsseldateien nicht gefunden werden können
        """
        # Pfade zu den Schlüsseldateien
        public_key_path = os.path.join(directory, "public_key.pem")
        private_key_path = os.path.join(directory, "private_key.pem")

        # Prüfe ob die Dateien existieren
        if not os.path.exists(public_key_path) or not os.path.exists(private_key_path):
            return "", ""

        # Hole den Device Key
        device_key = DEVICE_KEY()

        # Lade den öffentlichen Schlüssel
        with open(public_key_path) as f:
            public_key = f.read()

        # Lade und entschlüssele den privaten Schlüssel
        with open(private_key_path) as f:
            encrypted_private_key = f.read()
            private_key = Code.decrypt_symmetric(encrypted_private_key, device_key)

        return public_key, private_key

    @staticmethod
    def encrypt_asymmetric(text: str, public_key_str: str) -> str:
        """
        Verschlüsselt einen Text mit einem gegebenen öffentlichen Schlüssel.

        Args:
            text (str): Der zu verschlüsselnde Text.
            public_key_str (str): Der öffentliche Schlüssel als String oder im pem format.

        Returns:
            str: Der verschlüsselte Text.
        """
        # try:
        #    public_key: RSAPublicKey = serialization.load_pem_public_key(public_key_str.encode())
        #  except Exception as e:
        #     get_logger().error(f"Error encrypt_asymmetric {e}")
        try:
            public_key: RSAPublicKey = serialization.load_pem_public_key(public_key_str.encode())
            encrypted = public_key.encrypt(
                text.encode(),
                padding.OAEP(
                    mgf=padding.MGF1(algorithm=hashes.SHA512()),
                    algorithm=hashes.SHA512(),
                    label=None
                )
            )
            return encrypted.hex()
        except Exception as e:
            get_logger().error(f"Error encrypt_asymmetric {e}")
            return "Invalid"

    @staticmethod
    def decrypt_asymmetric(encrypted_text_hex: str, private_key_str: str) -> str:
        """
        Entschlüsselt einen Text mit einem gegebenen privaten Schlüssel.

        Args:
            encrypted_text_hex (str): Der verschlüsselte Text als Hex-String.
            private_key_str (str): Der private Schlüssel als String.

        Returns:
            str: Der entschlüsselte Text.
        """
        try:
            private_key = serialization.load_pem_private_key(private_key_str.encode(), password=None)
            decrypted = private_key.decrypt(
                bytes.fromhex(encrypted_text_hex),
                padding.OAEP(
                    mgf=padding.MGF1(algorithm=hashes.SHA512()),
                    algorithm=hashes.SHA512(),
                    label=None
                )
            )
            return decrypted.decode()

        except Exception as e:
            get_logger().error(f"Error decrypt_asymmetric {e}")
        return "Invalid"

    @staticmethod
    def verify_signature(signature: str or bytes, message: str or bytes, public_key_str: str,
                         salt_length=padding.PSS.MAX_LENGTH) -> bool:
        if isinstance(signature, str):
            signature = signature.encode()
        if isinstance(message, str):
            message = message.encode()
        try:
            public_key: RSAPublicKey = serialization.load_pem_public_key(public_key_str.encode())
            public_key.verify(
                signature=signature,
                data=message,
                padding=padding.PSS(
                    mgf=padding.MGF1(hashes.SHA512()),
                    salt_length=salt_length
                ),
                algorithm=hashes.SHA512()
            )
            return True
        except:
            pass
        return False

    @staticmethod
    def verify_signature_web_algo(signature: str or bytes, message: str or bytes, public_key_str: str,
                                  algo: int = -512) -> bool:
        signature_algorithm = ECDSA(hashes.SHA512())
        if algo != -512:
            signature_algorithm = ECDSA(hashes.SHA256())

        if isinstance(signature, str):
            signature = signature.encode()
        if isinstance(message, str):
            message = message.encode()
        try:
            public_key = serialization.load_pem_public_key(public_key_str.encode())
            public_key.verify(
                signature=signature,
                data=message,
                # padding=padding.PSS(
                #    mgf=padding.MGF1(hashes.SHA512()),
                #    salt_length=padding.PSS.MAX_LENGTH
                # ),
                signature_algorithm=signature_algorithm
            )
            return True
        except:
            pass
        return False

    @staticmethod
    def create_signature(message: str, private_key_str: str, salt_length=padding.PSS.MAX_LENGTH,
                         row=False) -> str or bytes:
        try:
            private_key = serialization.load_pem_private_key(private_key_str.encode(), password=None)
            signature = private_key.sign(
                message.encode(),
                padding.PSS(
                    mgf=padding.MGF1(hashes.SHA512()),
                    salt_length=salt_length
                ),
                hashes.SHA512()
            )
            if row:
                return signature
            return base64.b64encode(signature).decode()
        except Exception as e:
            get_logger().error(f"Error create_signature {e}")
            print(e)
        return "Invalid Key"

    @staticmethod
    def pem_to_public_key(pem_key: str):
        """
        Konvertiert einen PEM-kodierten öffentlichen Schlüssel in ein PublicKey-Objekt.

        Args:
            pem_key (str): Der PEM-kodierte öffentliche Schlüssel.

        Returns:
            PublicKey: Das PublicKey-Objekt.
        """
        public_key = serialization.load_pem_public_key(pem_key.encode())
        return public_key

    @staticmethod
    def public_key_to_pem(public_key: RSAPublicKey):
        """
        Konvertiert ein PublicKey-Objekt in einen PEM-kodierten String.

        Args:
            public_key (PublicKey): Das PublicKey-Objekt.

        Returns:
            str: Der PEM-kodierte öffentliche Schlüssel.
        """
        pem = public_key.public_bytes(
            encoding=serialization.Encoding.PEM,
            format=serialization.PublicFormat.SubjectPublicKeyInfo
        )
        return pem.decode()

decrypt_asymmetric(encrypted_text_hex, private_key_str) staticmethod

Entschlüsselt einen Text mit einem gegebenen privaten Schlüssel.

Parameters:

Name	Type	Description	Default
encrypted_text_hex	str	Der verschlüsselte Text als Hex-String.	required
private_key_str	str	Der private Schlüssel als String.	required

Returns:

Name	Type	Description
str	str	Der entschlüsselte Text.

Source code in toolboxv2/utils/security/cryp.py

@staticmethod
def decrypt_asymmetric(encrypted_text_hex: str, private_key_str: str) -> str:
    """
    Entschlüsselt einen Text mit einem gegebenen privaten Schlüssel.

    Args:
        encrypted_text_hex (str): Der verschlüsselte Text als Hex-String.
        private_key_str (str): Der private Schlüssel als String.

    Returns:
        str: Der entschlüsselte Text.
    """
    try:
        private_key = serialization.load_pem_private_key(private_key_str.encode(), password=None)
        decrypted = private_key.decrypt(
            bytes.fromhex(encrypted_text_hex),
            padding.OAEP(
                mgf=padding.MGF1(algorithm=hashes.SHA512()),
                algorithm=hashes.SHA512(),
                label=None
            )
        )
        return decrypted.decode()

    except Exception as e:
        get_logger().error(f"Error decrypt_asymmetric {e}")
    return "Invalid"

decrypt_symmetric(encrypted_text, key, to_str=True, mute=False) staticmethod

Entschlüsselt einen Text mit einem gegebenen symmetrischen Schlüssel.

Parameters:

Name	Type	Description	Default
encrypted_text	str	Der zu entschlüsselnde Text.	required
key	str	Der symmetrische Schlüssel.	required
to_str	bool	default true returns str if false returns bytes	True

Returns: str: Der entschlüsselte Text.

Source code in toolboxv2/utils/security/cryp.py

@staticmethod
def decrypt_symmetric(encrypted_text: str, key: str, to_str=True, mute=False) -> str or bytes:
    """
    Entschlüsselt einen Text mit einem gegebenen symmetrischen Schlüssel.

    Args:
        encrypted_text (str): Der zu entschlüsselnde Text.
        key (str): Der symmetrische Schlüssel.
        to_str (bool): default true returns str if false returns bytes
    Returns:
        str: Der entschlüsselte Text.
    """

    if isinstance(key, str):
        key = key.encode()

    #try:
    fernet = Fernet(key)
    text_b = fernet.decrypt(encrypted_text)
    if not to_str:
        return text_b
    return text_b.decode()

encrypt_asymmetric(text, public_key_str) staticmethod

Verschlüsselt einen Text mit einem gegebenen öffentlichen Schlüssel.

Parameters:

Name	Type	Description	Default
text	str	Der zu verschlüsselnde Text.	required
public_key_str	str	Der öffentliche Schlüssel als String oder im pem format.	required

Returns:

Name	Type	Description
str	str	Der verschlüsselte Text.

Source code in toolboxv2/utils/security/cryp.py

@staticmethod
def encrypt_asymmetric(text: str, public_key_str: str) -> str:
    """
    Verschlüsselt einen Text mit einem gegebenen öffentlichen Schlüssel.

    Args:
        text (str): Der zu verschlüsselnde Text.
        public_key_str (str): Der öffentliche Schlüssel als String oder im pem format.

    Returns:
        str: Der verschlüsselte Text.
    """
    # try:
    #    public_key: RSAPublicKey = serialization.load_pem_public_key(public_key_str.encode())
    #  except Exception as e:
    #     get_logger().error(f"Error encrypt_asymmetric {e}")
    try:
        public_key: RSAPublicKey = serialization.load_pem_public_key(public_key_str.encode())
        encrypted = public_key.encrypt(
            text.encode(),
            padding.OAEP(
                mgf=padding.MGF1(algorithm=hashes.SHA512()),
                algorithm=hashes.SHA512(),
                label=None
            )
        )
        return encrypted.hex()
    except Exception as e:
        get_logger().error(f"Error encrypt_asymmetric {e}")
        return "Invalid"

encrypt_symmetric(text, key) staticmethod

Verschlüsselt einen Text mit einem gegebenen symmetrischen Schlüssel.

Parameters:

Name	Type	Description	Default
text	str	Der zu verschlüsselnde Text.	required
key	str	Der symmetrische Schlüssel.	required

Returns:

Name	Type	Description
str	str	Der verschlüsselte Text.

Source code in toolboxv2/utils/security/cryp.py

@staticmethod
def encrypt_symmetric(text: str or bytes, key: str) -> str:
    """
    Verschlüsselt einen Text mit einem gegebenen symmetrischen Schlüssel.

    Args:
        text (str): Der zu verschlüsselnde Text.
        key (str): Der symmetrische Schlüssel.

    Returns:
        str: Der verschlüsselte Text.
    """
    if isinstance(text, str):
        text = text.encode()

    try:
        fernet = Fernet(key.encode())
        return fernet.encrypt(text).decode()
    except Exception as e:
        get_logger().error(f"Error encrypt_symmetric #{str(e)}#")
        return "Error encrypt"

generate_asymmetric_keys() staticmethod

Generiert ein Paar von öffentlichen und privaten Schlüsseln für die asymmetrische Verschlüsselung.

Parameters:

Name	Type	Description	Default
seed	int	Ein optionaler Seed-Wert. Standardmäßig None.	required

Returns:

Type	Description
(str, str)	Ein Tupel aus öffentlichem und privatem Schlüssel.

Source code in toolboxv2/utils/security/cryp.py

@staticmethod
def generate_asymmetric_keys() -> (str, str):
    """
    Generiert ein Paar von öffentlichen und privaten Schlüsseln für die asymmetrische Verschlüsselung.

    Args:
        seed (int, optional): Ein optionaler Seed-Wert. Standardmäßig None.

    Returns:
        (str, str): Ein Tupel aus öffentlichem und privatem Schlüssel.
    """
    private_key = rsa.generate_private_key(
        public_exponent=65537,
        key_size=2048 * 3,
    )
    public_key = private_key.public_key()

    # Serialisieren der Schlüssel
    pem_private_key = private_key.private_bytes(
        encoding=serialization.Encoding.PEM,
        format=serialization.PrivateFormat.PKCS8,
        encryption_algorithm=serialization.NoEncryption()
    ).decode()

    pem_public_key = public_key.public_bytes(
        encoding=serialization.Encoding.PEM,
        format=serialization.PublicFormat.SubjectPublicKeyInfo
    ).decode()

    return pem_public_key, pem_private_key

generate_random_string(length) staticmethod

Generiert eine zufällige Zeichenkette der angegebenen Länge.

Parameters:

Name	Type	Description	Default
length	int	Die Länge der zu generierenden Zeichenkette.	required

Returns:

Name	Type	Description
str	str	Die generierte Zeichenkette.

Source code in toolboxv2/utils/security/cryp.py

@staticmethod
def generate_random_string(length: int) -> str:
    """
    Generiert eine zufällige Zeichenkette der angegebenen Länge.

    Args:
        length (int): Die Länge der zu generierenden Zeichenkette.

    Returns:
        str: Die generierte Zeichenkette.
    """
    return secrets.token_urlsafe(length)

generate_seed() staticmethod

Erzeugt eine zufällige Zahl als Seed.

Returns:

Name	Type	Description
int	int	Eine zufällige Zahl.

Source code in toolboxv2/utils/security/cryp.py

@staticmethod
def generate_seed() -> int:
    """
    Erzeugt eine zufällige Zahl als Seed.

    Returns:
        int: Eine zufällige Zahl.
    """
    return random.randint(2 ** 32 - 1, 2 ** 64 - 1)

generate_symmetric_key(as_str=True) staticmethod

Generiert einen Schlüssel für die symmetrische Verschlüsselung.

Returns:

Name	Type	Description
str	str or bytes	Der generierte Schlüssel.

Source code in toolboxv2/utils/security/cryp.py

@staticmethod
def generate_symmetric_key(as_str=True) -> str or bytes:
    """
    Generiert einen Schlüssel für die symmetrische Verschlüsselung.

    Returns:
        str: Der generierte Schlüssel.
    """
    key = Fernet.generate_key()
    if as_str:
        key = key.decode()
    return key

load_keys_from_files(directory='keys') staticmethod

Lädt die Schlüssel aus den Dateien. Der private Schlüssel wird mit dem Device Key entschlüsselt.

Parameters:

Name	Type	Description	Default
directory	str	Das Verzeichnis, aus dem die Schlüssel geladen werden sollen	'keys'

Returns:

Type	Description
(str, str)	Ein Tupel aus öffentlichem und privatem Schlüssel

Raises:

Type	Description
FileNotFoundError	Wenn die Schlüsseldateien nicht gefunden werden können

Source code in toolboxv2/utils/security/cryp.py

@staticmethod
def load_keys_from_files(directory: str = "keys") -> (str, str):
    """
    Lädt die Schlüssel aus den Dateien.
    Der private Schlüssel wird mit dem Device Key entschlüsselt.

    Args:
        directory (str): Das Verzeichnis, aus dem die Schlüssel geladen werden sollen

    Returns:
        (str, str): Ein Tupel aus öffentlichem und privatem Schlüssel

    Raises:
        FileNotFoundError: Wenn die Schlüsseldateien nicht gefunden werden können
    """
    # Pfade zu den Schlüsseldateien
    public_key_path = os.path.join(directory, "public_key.pem")
    private_key_path = os.path.join(directory, "private_key.pem")

    # Prüfe ob die Dateien existieren
    if not os.path.exists(public_key_path) or not os.path.exists(private_key_path):
        return "", ""

    # Hole den Device Key
    device_key = DEVICE_KEY()

    # Lade den öffentlichen Schlüssel
    with open(public_key_path) as f:
        public_key = f.read()

    # Lade und entschlüssele den privaten Schlüssel
    with open(private_key_path) as f:
        encrypted_private_key = f.read()
        private_key = Code.decrypt_symmetric(encrypted_private_key, device_key)

    return public_key, private_key

one_way_hash(text, salt='', pepper='') staticmethod

Erzeugt einen Hash eines gegebenen Textes mit Salt, Pepper und optional einem Seed.

Parameters:

Name	Type	Description	Default
text	str	Der zu hashende Text.	required
salt	str	Der Salt-Wert.	''
pepper	str	Der Pepper-Wert.	''
seed	int	Ein optionaler Seed-Wert. Standardmäßig None.	required

Returns:

Name	Type	Description
str	str	Der resultierende Hash-Wert.

Source code in toolboxv2/utils/security/cryp.py

@staticmethod
def one_way_hash(text: str, salt: str = '', pepper: str = '') -> str:
    """
    Erzeugt einen Hash eines gegebenen Textes mit Salt, Pepper und optional einem Seed.

    Args:
        text (str): Der zu hashende Text.
        salt (str): Der Salt-Wert.
        pepper (str): Der Pepper-Wert.
        seed (int, optional): Ein optionaler Seed-Wert. Standardmäßig None.

    Returns:
        str: Der resultierende Hash-Wert.
    """
    return hashlib.sha256((salt + text + pepper).encode()).hexdigest()

pem_to_public_key(pem_key) staticmethod

Konvertiert einen PEM-kodierten öffentlichen Schlüssel in ein PublicKey-Objekt.

Parameters:

Name	Type	Description	Default
pem_key	str	Der PEM-kodierte öffentliche Schlüssel.	required

Returns:

Name	Type	Description
PublicKey		Das PublicKey-Objekt.

Source code in toolboxv2/utils/security/cryp.py

@staticmethod
def pem_to_public_key(pem_key: str):
    """
    Konvertiert einen PEM-kodierten öffentlichen Schlüssel in ein PublicKey-Objekt.

    Args:
        pem_key (str): Der PEM-kodierte öffentliche Schlüssel.

    Returns:
        PublicKey: Das PublicKey-Objekt.
    """
    public_key = serialization.load_pem_public_key(pem_key.encode())
    return public_key

public_key_to_pem(public_key) staticmethod

Konvertiert ein PublicKey-Objekt in einen PEM-kodierten String.

Parameters:

Name	Type	Description	Default
public_key	PublicKey	Das PublicKey-Objekt.	required

Returns:

Name	Type	Description
str		Der PEM-kodierte öffentliche Schlüssel.

Source code in toolboxv2/utils/security/cryp.py

@staticmethod
def public_key_to_pem(public_key: RSAPublicKey):
    """
    Konvertiert ein PublicKey-Objekt in einen PEM-kodierten String.

    Args:
        public_key (PublicKey): Das PublicKey-Objekt.

    Returns:
        str: Der PEM-kodierte öffentliche Schlüssel.
    """
    pem = public_key.public_bytes(
        encoding=serialization.Encoding.PEM,
        format=serialization.PublicFormat.SubjectPublicKeyInfo
    )
    return pem.decode()

save_keys_to_files(public_key, private_key, directory='keys') staticmethod

Speichert die generierten Schlüssel in separate Dateien. Der private Schlüssel wird mit dem Device Key verschlüsselt.

Parameters:

Name	Type	Description	Default
public_key	str	Der öffentliche Schlüssel im PEM-Format	required
private_key	str	Der private Schlüssel im PEM-Format	required
directory	str	Das Verzeichnis, in dem die Schlüssel gespeichert werden sollen	'keys'

Source code in toolboxv2/utils/security/cryp.py

@staticmethod
def save_keys_to_files(public_key: str, private_key: str, directory: str = "keys") -> None:
    """
    Speichert die generierten Schlüssel in separate Dateien.
    Der private Schlüssel wird mit dem Device Key verschlüsselt.

    Args:
        public_key (str): Der öffentliche Schlüssel im PEM-Format
        private_key (str): Der private Schlüssel im PEM-Format
        directory (str): Das Verzeichnis, in dem die Schlüssel gespeichert werden sollen
    """
    # Erstelle das Verzeichnis, falls es nicht existiert
    os.makedirs(directory, exist_ok=True)

    # Hole den Device Key
    device_key = DEVICE_KEY()

    # Verschlüssele den privaten Schlüssel mit dem Device Key
    encrypted_private_key = Code.encrypt_symmetric(private_key, device_key)

    # Speichere den öffentlichen Schlüssel
    public_key_path = os.path.join(directory, "public_key.pem")
    with open(public_key_path, "w") as f:
        f.write(public_key)

    # Speichere den verschlüsselten privaten Schlüssel
    private_key_path = os.path.join(directory, "private_key.pem")
    with open(private_key_path, "w") as f:
        f.write(encrypted_private_key)

    print("Saved keys in ", public_key_path)

cryp

Code

Source code in toolboxv2/utils/security/cryp.py

class Code:

    @staticmethod
    def DK():
        return DEVICE_KEY

    @staticmethod
    def generate_random_string(length: int) -> str:
        """
        Generiert eine zufällige Zeichenkette der angegebenen Länge.

        Args:
            length (int): Die Länge der zu generierenden Zeichenkette.

        Returns:
            str: Die generierte Zeichenkette.
        """
        return secrets.token_urlsafe(length)

    def decode_code(self, encrypted_data, key=None):

        if not isinstance(encrypted_data, str):
            encrypted_data = str(encrypted_data)

        if key is None:
            key = DEVICE_KEY()

        return self.decrypt_symmetric(encrypted_data, key)

    def encode_code(self, data, key=None):

        if not isinstance(data, str):
            data = str(data)

        if key is None:
            key = DEVICE_KEY()

        return self.encrypt_symmetric(data, key)

    @staticmethod
    def generate_seed() -> int:
        """
        Erzeugt eine zufällige Zahl als Seed.

        Returns:
            int: Eine zufällige Zahl.
        """
        return random.randint(2 ** 32 - 1, 2 ** 64 - 1)

    @staticmethod
    def one_way_hash(text: str, salt: str = '', pepper: str = '') -> str:
        """
        Erzeugt einen Hash eines gegebenen Textes mit Salt, Pepper und optional einem Seed.

        Args:
            text (str): Der zu hashende Text.
            salt (str): Der Salt-Wert.
            pepper (str): Der Pepper-Wert.
            seed (int, optional): Ein optionaler Seed-Wert. Standardmäßig None.

        Returns:
            str: Der resultierende Hash-Wert.
        """
        return hashlib.sha256((salt + text + pepper).encode()).hexdigest()

    @staticmethod
    def generate_symmetric_key(as_str=True) -> str or bytes:
        """
        Generiert einen Schlüssel für die symmetrische Verschlüsselung.

        Returns:
            str: Der generierte Schlüssel.
        """
        key = Fernet.generate_key()
        if as_str:
            key = key.decode()
        return key

    @staticmethod
    def encrypt_symmetric(text: str or bytes, key: str) -> str:
        """
        Verschlüsselt einen Text mit einem gegebenen symmetrischen Schlüssel.

        Args:
            text (str): Der zu verschlüsselnde Text.
            key (str): Der symmetrische Schlüssel.

        Returns:
            str: Der verschlüsselte Text.
        """
        if isinstance(text, str):
            text = text.encode()

        try:
            fernet = Fernet(key.encode())
            return fernet.encrypt(text).decode()
        except Exception as e:
            get_logger().error(f"Error encrypt_symmetric #{str(e)}#")
            return "Error encrypt"

    @staticmethod
    def decrypt_symmetric(encrypted_text: str, key: str, to_str=True, mute=False) -> str or bytes:
        """
        Entschlüsselt einen Text mit einem gegebenen symmetrischen Schlüssel.

        Args:
            encrypted_text (str): Der zu entschlüsselnde Text.
            key (str): Der symmetrische Schlüssel.
            to_str (bool): default true returns str if false returns bytes
        Returns:
            str: Der entschlüsselte Text.
        """

        if isinstance(key, str):
            key = key.encode()

        #try:
        fernet = Fernet(key)
        text_b = fernet.decrypt(encrypted_text)
        if not to_str:
            return text_b
        return text_b.decode()
        # except Exception as e:
        #     get_logger().error(f"Error decrypt_symmetric {e}")
        #     if not mute:
        #         raise e
        #     if not to_str:
        #         return f"Error decoding".encode()
        #     return f"Error decoding"

    @staticmethod
    def generate_asymmetric_keys() -> (str, str):
        """
        Generiert ein Paar von öffentlichen und privaten Schlüsseln für die asymmetrische Verschlüsselung.

        Args:
            seed (int, optional): Ein optionaler Seed-Wert. Standardmäßig None.

        Returns:
            (str, str): Ein Tupel aus öffentlichem und privatem Schlüssel.
        """
        private_key = rsa.generate_private_key(
            public_exponent=65537,
            key_size=2048 * 3,
        )
        public_key = private_key.public_key()

        # Serialisieren der Schlüssel
        pem_private_key = private_key.private_bytes(
            encoding=serialization.Encoding.PEM,
            format=serialization.PrivateFormat.PKCS8,
            encryption_algorithm=serialization.NoEncryption()
        ).decode()

        pem_public_key = public_key.public_bytes(
            encoding=serialization.Encoding.PEM,
            format=serialization.PublicFormat.SubjectPublicKeyInfo
        ).decode()

        return pem_public_key, pem_private_key

    @staticmethod
    def save_keys_to_files(public_key: str, private_key: str, directory: str = "keys") -> None:
        """
        Speichert die generierten Schlüssel in separate Dateien.
        Der private Schlüssel wird mit dem Device Key verschlüsselt.

        Args:
            public_key (str): Der öffentliche Schlüssel im PEM-Format
            private_key (str): Der private Schlüssel im PEM-Format
            directory (str): Das Verzeichnis, in dem die Schlüssel gespeichert werden sollen
        """
        # Erstelle das Verzeichnis, falls es nicht existiert
        os.makedirs(directory, exist_ok=True)

        # Hole den Device Key
        device_key = DEVICE_KEY()

        # Verschlüssele den privaten Schlüssel mit dem Device Key
        encrypted_private_key = Code.encrypt_symmetric(private_key, device_key)

        # Speichere den öffentlichen Schlüssel
        public_key_path = os.path.join(directory, "public_key.pem")
        with open(public_key_path, "w") as f:
            f.write(public_key)

        # Speichere den verschlüsselten privaten Schlüssel
        private_key_path = os.path.join(directory, "private_key.pem")
        with open(private_key_path, "w") as f:
            f.write(encrypted_private_key)

        print("Saved keys in ", public_key_path)

    @staticmethod
    def load_keys_from_files(directory: str = "keys") -> (str, str):
        """
        Lädt die Schlüssel aus den Dateien.
        Der private Schlüssel wird mit dem Device Key entschlüsselt.

        Args:
            directory (str): Das Verzeichnis, aus dem die Schlüssel geladen werden sollen

        Returns:
            (str, str): Ein Tupel aus öffentlichem und privatem Schlüssel

        Raises:
            FileNotFoundError: Wenn die Schlüsseldateien nicht gefunden werden können
        """
        # Pfade zu den Schlüsseldateien
        public_key_path = os.path.join(directory, "public_key.pem")
        private_key_path = os.path.join(directory, "private_key.pem")

        # Prüfe ob die Dateien existieren
        if not os.path.exists(public_key_path) or not os.path.exists(private_key_path):
            return "", ""

        # Hole den Device Key
        device_key = DEVICE_KEY()

        # Lade den öffentlichen Schlüssel
        with open(public_key_path) as f:
            public_key = f.read()

        # Lade und entschlüssele den privaten Schlüssel
        with open(private_key_path) as f:
            encrypted_private_key = f.read()
            private_key = Code.decrypt_symmetric(encrypted_private_key, device_key)

        return public_key, private_key

    @staticmethod
    def encrypt_asymmetric(text: str, public_key_str: str) -> str:
        """
        Verschlüsselt einen Text mit einem gegebenen öffentlichen Schlüssel.

        Args:
            text (str): Der zu verschlüsselnde Text.
            public_key_str (str): Der öffentliche Schlüssel als String oder im pem format.

        Returns:
            str: Der verschlüsselte Text.
        """
        # try:
        #    public_key: RSAPublicKey = serialization.load_pem_public_key(public_key_str.encode())
        #  except Exception as e:
        #     get_logger().error(f"Error encrypt_asymmetric {e}")
        try:
            public_key: RSAPublicKey = serialization.load_pem_public_key(public_key_str.encode())
            encrypted = public_key.encrypt(
                text.encode(),
                padding.OAEP(
                    mgf=padding.MGF1(algorithm=hashes.SHA512()),
                    algorithm=hashes.SHA512(),
                    label=None
                )
            )
            return encrypted.hex()
        except Exception as e:
            get_logger().error(f"Error encrypt_asymmetric {e}")
            return "Invalid"

    @staticmethod
    def decrypt_asymmetric(encrypted_text_hex: str, private_key_str: str) -> str:
        """
        Entschlüsselt einen Text mit einem gegebenen privaten Schlüssel.

        Args:
            encrypted_text_hex (str): Der verschlüsselte Text als Hex-String.
            private_key_str (str): Der private Schlüssel als String.

        Returns:
            str: Der entschlüsselte Text.
        """
        try:
            private_key = serialization.load_pem_private_key(private_key_str.encode(), password=None)
            decrypted = private_key.decrypt(
                bytes.fromhex(encrypted_text_hex),
                padding.OAEP(
                    mgf=padding.MGF1(algorithm=hashes.SHA512()),
                    algorithm=hashes.SHA512(),
                    label=None
                )
            )
            return decrypted.decode()

        except Exception as e:
            get_logger().error(f"Error decrypt_asymmetric {e}")
        return "Invalid"

    @staticmethod
    def verify_signature(signature: str or bytes, message: str or bytes, public_key_str: str,
                         salt_length=padding.PSS.MAX_LENGTH) -> bool:
        if isinstance(signature, str):
            signature = signature.encode()
        if isinstance(message, str):
            message = message.encode()
        try:
            public_key: RSAPublicKey = serialization.load_pem_public_key(public_key_str.encode())
            public_key.verify(
                signature=signature,
                data=message,
                padding=padding.PSS(
                    mgf=padding.MGF1(hashes.SHA512()),
                    salt_length=salt_length
                ),
                algorithm=hashes.SHA512()
            )
            return True
        except:
            pass
        return False

    @staticmethod
    def verify_signature_web_algo(signature: str or bytes, message: str or bytes, public_key_str: str,
                                  algo: int = -512) -> bool:
        signature_algorithm = ECDSA(hashes.SHA512())
        if algo != -512:
            signature_algorithm = ECDSA(hashes.SHA256())

        if isinstance(signature, str):
            signature = signature.encode()
        if isinstance(message, str):
            message = message.encode()
        try:
            public_key = serialization.load_pem_public_key(public_key_str.encode())
            public_key.verify(
                signature=signature,
                data=message,
                # padding=padding.PSS(
                #    mgf=padding.MGF1(hashes.SHA512()),
                #    salt_length=padding.PSS.MAX_LENGTH
                # ),
                signature_algorithm=signature_algorithm
            )
            return True
        except:
            pass
        return False

    @staticmethod
    def create_signature(message: str, private_key_str: str, salt_length=padding.PSS.MAX_LENGTH,
                         row=False) -> str or bytes:
        try:
            private_key = serialization.load_pem_private_key(private_key_str.encode(), password=None)
            signature = private_key.sign(
                message.encode(),
                padding.PSS(
                    mgf=padding.MGF1(hashes.SHA512()),
                    salt_length=salt_length
                ),
                hashes.SHA512()
            )
            if row:
                return signature
            return base64.b64encode(signature).decode()
        except Exception as e:
            get_logger().error(f"Error create_signature {e}")
            print(e)
        return "Invalid Key"

    @staticmethod
    def pem_to_public_key(pem_key: str):
        """
        Konvertiert einen PEM-kodierten öffentlichen Schlüssel in ein PublicKey-Objekt.

        Args:
            pem_key (str): Der PEM-kodierte öffentliche Schlüssel.

        Returns:
            PublicKey: Das PublicKey-Objekt.
        """
        public_key = serialization.load_pem_public_key(pem_key.encode())
        return public_key

    @staticmethod
    def public_key_to_pem(public_key: RSAPublicKey):
        """
        Konvertiert ein PublicKey-Objekt in einen PEM-kodierten String.

        Args:
            public_key (PublicKey): Das PublicKey-Objekt.

        Returns:
            str: Der PEM-kodierte öffentliche Schlüssel.
        """
        pem = public_key.public_bytes(
            encoding=serialization.Encoding.PEM,
            format=serialization.PublicFormat.SubjectPublicKeyInfo
        )
        return pem.decode()

decrypt_asymmetric(encrypted_text_hex, private_key_str) staticmethod

Entschlüsselt einen Text mit einem gegebenen privaten Schlüssel.

Parameters:

Name	Type	Description	Default
encrypted_text_hex	str	Der verschlüsselte Text als Hex-String.	required
private_key_str	str	Der private Schlüssel als String.	required

Returns:

Name	Type	Description
str	str	Der entschlüsselte Text.

Source code in toolboxv2/utils/security/cryp.py

@staticmethod
def decrypt_asymmetric(encrypted_text_hex: str, private_key_str: str) -> str:
    """
    Entschlüsselt einen Text mit einem gegebenen privaten Schlüssel.

    Args:
        encrypted_text_hex (str): Der verschlüsselte Text als Hex-String.
        private_key_str (str): Der private Schlüssel als String.

    Returns:
        str: Der entschlüsselte Text.
    """
    try:
        private_key = serialization.load_pem_private_key(private_key_str.encode(), password=None)
        decrypted = private_key.decrypt(
            bytes.fromhex(encrypted_text_hex),
            padding.OAEP(
                mgf=padding.MGF1(algorithm=hashes.SHA512()),
                algorithm=hashes.SHA512(),
                label=None
            )
        )
        return decrypted.decode()

    except Exception as e:
        get_logger().error(f"Error decrypt_asymmetric {e}")
    return "Invalid"

decrypt_symmetric(encrypted_text, key, to_str=True, mute=False) staticmethod

Entschlüsselt einen Text mit einem gegebenen symmetrischen Schlüssel.

Parameters:

Name	Type	Description	Default
encrypted_text	str	Der zu entschlüsselnde Text.	required
key	str	Der symmetrische Schlüssel.	required
to_str	bool	default true returns str if false returns bytes	True

Returns: str: Der entschlüsselte Text.

Source code in toolboxv2/utils/security/cryp.py

@staticmethod
def decrypt_symmetric(encrypted_text: str, key: str, to_str=True, mute=False) -> str or bytes:
    """
    Entschlüsselt einen Text mit einem gegebenen symmetrischen Schlüssel.

    Args:
        encrypted_text (str): Der zu entschlüsselnde Text.
        key (str): Der symmetrische Schlüssel.
        to_str (bool): default true returns str if false returns bytes
    Returns:
        str: Der entschlüsselte Text.
    """

    if isinstance(key, str):
        key = key.encode()

    #try:
    fernet = Fernet(key)
    text_b = fernet.decrypt(encrypted_text)
    if not to_str:
        return text_b
    return text_b.decode()

encrypt_asymmetric(text, public_key_str) staticmethod

Verschlüsselt einen Text mit einem gegebenen öffentlichen Schlüssel.

Parameters:

Name	Type	Description	Default
text	str	Der zu verschlüsselnde Text.	required
public_key_str	str	Der öffentliche Schlüssel als String oder im pem format.	required

Returns:

Name	Type	Description
str	str	Der verschlüsselte Text.

Source code in toolboxv2/utils/security/cryp.py

@staticmethod
def encrypt_asymmetric(text: str, public_key_str: str) -> str:
    """
    Verschlüsselt einen Text mit einem gegebenen öffentlichen Schlüssel.

    Args:
        text (str): Der zu verschlüsselnde Text.
        public_key_str (str): Der öffentliche Schlüssel als String oder im pem format.

    Returns:
        str: Der verschlüsselte Text.
    """
    # try:
    #    public_key: RSAPublicKey = serialization.load_pem_public_key(public_key_str.encode())
    #  except Exception as e:
    #     get_logger().error(f"Error encrypt_asymmetric {e}")
    try:
        public_key: RSAPublicKey = serialization.load_pem_public_key(public_key_str.encode())
        encrypted = public_key.encrypt(
            text.encode(),
            padding.OAEP(
                mgf=padding.MGF1(algorithm=hashes.SHA512()),
                algorithm=hashes.SHA512(),
                label=None
            )
        )
        return encrypted.hex()
    except Exception as e:
        get_logger().error(f"Error encrypt_asymmetric {e}")
        return "Invalid"

encrypt_symmetric(text, key) staticmethod

Verschlüsselt einen Text mit einem gegebenen symmetrischen Schlüssel.

Parameters:

Name	Type	Description	Default
text	str	Der zu verschlüsselnde Text.	required
key	str	Der symmetrische Schlüssel.	required

Returns:

Name	Type	Description
str	str	Der verschlüsselte Text.

Source code in toolboxv2/utils/security/cryp.py

@staticmethod
def encrypt_symmetric(text: str or bytes, key: str) -> str:
    """
    Verschlüsselt einen Text mit einem gegebenen symmetrischen Schlüssel.

    Args:
        text (str): Der zu verschlüsselnde Text.
        key (str): Der symmetrische Schlüssel.

    Returns:
        str: Der verschlüsselte Text.
    """
    if isinstance(text, str):
        text = text.encode()

    try:
        fernet = Fernet(key.encode())
        return fernet.encrypt(text).decode()
    except Exception as e:
        get_logger().error(f"Error encrypt_symmetric #{str(e)}#")
        return "Error encrypt"

generate_asymmetric_keys() staticmethod

Generiert ein Paar von öffentlichen und privaten Schlüsseln für die asymmetrische Verschlüsselung.

Parameters:

Name	Type	Description	Default
seed	int	Ein optionaler Seed-Wert. Standardmäßig None.	required

Returns:

Type	Description
(str, str)	Ein Tupel aus öffentlichem und privatem Schlüssel.

Source code in toolboxv2/utils/security/cryp.py

@staticmethod
def generate_asymmetric_keys() -> (str, str):
    """
    Generiert ein Paar von öffentlichen und privaten Schlüsseln für die asymmetrische Verschlüsselung.

    Args:
        seed (int, optional): Ein optionaler Seed-Wert. Standardmäßig None.

    Returns:
        (str, str): Ein Tupel aus öffentlichem und privatem Schlüssel.
    """
    private_key = rsa.generate_private_key(
        public_exponent=65537,
        key_size=2048 * 3,
    )
    public_key = private_key.public_key()

    # Serialisieren der Schlüssel
    pem_private_key = private_key.private_bytes(
        encoding=serialization.Encoding.PEM,
        format=serialization.PrivateFormat.PKCS8,
        encryption_algorithm=serialization.NoEncryption()
    ).decode()

    pem_public_key = public_key.public_bytes(
        encoding=serialization.Encoding.PEM,
        format=serialization.PublicFormat.SubjectPublicKeyInfo
    ).decode()

    return pem_public_key, pem_private_key

generate_random_string(length) staticmethod

Generiert eine zufällige Zeichenkette der angegebenen Länge.

Parameters:

Name	Type	Description	Default
length	int	Die Länge der zu generierenden Zeichenkette.	required

Returns:

Name	Type	Description
str	str	Die generierte Zeichenkette.

Source code in toolboxv2/utils/security/cryp.py

@staticmethod
def generate_random_string(length: int) -> str:
    """
    Generiert eine zufällige Zeichenkette der angegebenen Länge.

    Args:
        length (int): Die Länge der zu generierenden Zeichenkette.

    Returns:
        str: Die generierte Zeichenkette.
    """
    return secrets.token_urlsafe(length)

generate_seed() staticmethod

Erzeugt eine zufällige Zahl als Seed.

Returns:

Name	Type	Description
int	int	Eine zufällige Zahl.

Source code in toolboxv2/utils/security/cryp.py

@staticmethod
def generate_seed() -> int:
    """
    Erzeugt eine zufällige Zahl als Seed.

    Returns:
        int: Eine zufällige Zahl.
    """
    return random.randint(2 ** 32 - 1, 2 ** 64 - 1)

generate_symmetric_key(as_str=True) staticmethod

Generiert einen Schlüssel für die symmetrische Verschlüsselung.

Returns:

Name	Type	Description
str	str or bytes	Der generierte Schlüssel.

Source code in toolboxv2/utils/security/cryp.py

@staticmethod
def generate_symmetric_key(as_str=True) -> str or bytes:
    """
    Generiert einen Schlüssel für die symmetrische Verschlüsselung.

    Returns:
        str: Der generierte Schlüssel.
    """
    key = Fernet.generate_key()
    if as_str:
        key = key.decode()
    return key

load_keys_from_files(directory='keys') staticmethod

Lädt die Schlüssel aus den Dateien. Der private Schlüssel wird mit dem Device Key entschlüsselt.

Parameters:

Name	Type	Description	Default
directory	str	Das Verzeichnis, aus dem die Schlüssel geladen werden sollen	'keys'

Returns:

Type	Description
(str, str)	Ein Tupel aus öffentlichem und privatem Schlüssel

Raises:

Type	Description
FileNotFoundError	Wenn die Schlüsseldateien nicht gefunden werden können

Source code in toolboxv2/utils/security/cryp.py

@staticmethod
def load_keys_from_files(directory: str = "keys") -> (str, str):
    """
    Lädt die Schlüssel aus den Dateien.
    Der private Schlüssel wird mit dem Device Key entschlüsselt.

    Args:
        directory (str): Das Verzeichnis, aus dem die Schlüssel geladen werden sollen

    Returns:
        (str, str): Ein Tupel aus öffentlichem und privatem Schlüssel

    Raises:
        FileNotFoundError: Wenn die Schlüsseldateien nicht gefunden werden können
    """
    # Pfade zu den Schlüsseldateien
    public_key_path = os.path.join(directory, "public_key.pem")
    private_key_path = os.path.join(directory, "private_key.pem")

    # Prüfe ob die Dateien existieren
    if not os.path.exists(public_key_path) or not os.path.exists(private_key_path):
        return "", ""

    # Hole den Device Key
    device_key = DEVICE_KEY()

    # Lade den öffentlichen Schlüssel
    with open(public_key_path) as f:
        public_key = f.read()

    # Lade und entschlüssele den privaten Schlüssel
    with open(private_key_path) as f:
        encrypted_private_key = f.read()
        private_key = Code.decrypt_symmetric(encrypted_private_key, device_key)

    return public_key, private_key

one_way_hash(text, salt='', pepper='') staticmethod

Erzeugt einen Hash eines gegebenen Textes mit Salt, Pepper und optional einem Seed.

Parameters:

Name	Type	Description	Default
text	str	Der zu hashende Text.	required
salt	str	Der Salt-Wert.	''
pepper	str	Der Pepper-Wert.	''
seed	int	Ein optionaler Seed-Wert. Standardmäßig None.	required

Returns:

Name	Type	Description
str	str	Der resultierende Hash-Wert.

Source code in toolboxv2/utils/security/cryp.py

@staticmethod
def one_way_hash(text: str, salt: str = '', pepper: str = '') -> str:
    """
    Erzeugt einen Hash eines gegebenen Textes mit Salt, Pepper und optional einem Seed.

    Args:
        text (str): Der zu hashende Text.
        salt (str): Der Salt-Wert.
        pepper (str): Der Pepper-Wert.
        seed (int, optional): Ein optionaler Seed-Wert. Standardmäßig None.

    Returns:
        str: Der resultierende Hash-Wert.
    """
    return hashlib.sha256((salt + text + pepper).encode()).hexdigest()

pem_to_public_key(pem_key) staticmethod

Konvertiert einen PEM-kodierten öffentlichen Schlüssel in ein PublicKey-Objekt.

Parameters:

Name	Type	Description	Default
pem_key	str	Der PEM-kodierte öffentliche Schlüssel.	required

Returns:

Name	Type	Description
PublicKey		Das PublicKey-Objekt.

Source code in toolboxv2/utils/security/cryp.py

@staticmethod
def pem_to_public_key(pem_key: str):
    """
    Konvertiert einen PEM-kodierten öffentlichen Schlüssel in ein PublicKey-Objekt.

    Args:
        pem_key (str): Der PEM-kodierte öffentliche Schlüssel.

    Returns:
        PublicKey: Das PublicKey-Objekt.
    """
    public_key = serialization.load_pem_public_key(pem_key.encode())
    return public_key

public_key_to_pem(public_key) staticmethod

Konvertiert ein PublicKey-Objekt in einen PEM-kodierten String.

Parameters:

Name	Type	Description	Default
public_key	PublicKey	Das PublicKey-Objekt.	required

Returns:

Name	Type	Description
str		Der PEM-kodierte öffentliche Schlüssel.

Source code in toolboxv2/utils/security/cryp.py

@staticmethod
def public_key_to_pem(public_key: RSAPublicKey):
    """
    Konvertiert ein PublicKey-Objekt in einen PEM-kodierten String.

    Args:
        public_key (PublicKey): Das PublicKey-Objekt.

    Returns:
        str: Der PEM-kodierte öffentliche Schlüssel.
    """
    pem = public_key.public_bytes(
        encoding=serialization.Encoding.PEM,
        format=serialization.PublicFormat.SubjectPublicKeyInfo
    )
    return pem.decode()

save_keys_to_files(public_key, private_key, directory='keys') staticmethod

Speichert die generierten Schlüssel in separate Dateien. Der private Schlüssel wird mit dem Device Key verschlüsselt.

Parameters:

Name	Type	Description	Default
public_key	str	Der öffentliche Schlüssel im PEM-Format	required
private_key	str	Der private Schlüssel im PEM-Format	required
directory	str	Das Verzeichnis, in dem die Schlüssel gespeichert werden sollen	'keys'

Source code in toolboxv2/utils/security/cryp.py

@staticmethod
def save_keys_to_files(public_key: str, private_key: str, directory: str = "keys") -> None:
    """
    Speichert die generierten Schlüssel in separate Dateien.
    Der private Schlüssel wird mit dem Device Key verschlüsselt.

    Args:
        public_key (str): Der öffentliche Schlüssel im PEM-Format
        private_key (str): Der private Schlüssel im PEM-Format
        directory (str): Das Verzeichnis, in dem die Schlüssel gespeichert werden sollen
    """
    # Erstelle das Verzeichnis, falls es nicht existiert
    os.makedirs(directory, exist_ok=True)

    # Hole den Device Key
    device_key = DEVICE_KEY()

    # Verschlüssele den privaten Schlüssel mit dem Device Key
    encrypted_private_key = Code.encrypt_symmetric(private_key, device_key)

    # Speichere den öffentlichen Schlüssel
    public_key_path = os.path.join(directory, "public_key.pem")
    with open(public_key_path, "w") as f:
        f.write(public_key)

    # Speichere den verschlüsselten privaten Schlüssel
    private_key_path = os.path.join(directory, "private_key.pem")
    with open(private_key_path, "w") as f:
        f.write(encrypted_private_key)

    print("Saved keys in ", public_key_path)

singelton_class

Singleton

Singleton metaclass for ensuring only one instance of a class.

Source code in toolboxv2/utils/singelton_class.py

class Singleton(type):
    """
    Singleton metaclass for ensuring only one instance of a class.
    """

    _instances = {}
    _kwargs = {}
    _args = {}

    def __call__(cls, *args, **kwargs):
        if cls not in cls._instances:
            cls._instances[cls] = super().__call__(*args, **kwargs)
            cls._args[cls] = args
            cls._kwargs[cls] = kwargs
        return cls._instances[cls]

system

AppType

Source code in toolboxv2/utils/system/types.py

class AppType:
    prefix: str
    id: str
    globals: dict[str, Any] = {"root": dict, }
    locals: dict[str, Any] = {"user": {'app': "self"}, }

    local_test: bool = False
    start_dir: str
    data_dir: str
    config_dir: str
    info_dir: str
    is_server:bool = False

    logger: logging.Logger
    logging_filename: str

    api_allowed_mods_list: list[str] = []

    version: str
    loop: asyncio.AbstractEventLoop

    keys: dict[str, str] = {
        "MACRO": "macro~~~~:",
        "MACRO_C": "m_color~~:",
        "HELPER": "helper~~~:",
        "debug": "debug~~~~:",
        "id": "name-spa~:",
        "st-load": "mute~load:",
        "comm-his": "comm-his~:",
        "develop-mode": "dev~mode~:",
        "provider::": "provider::",
    }

    defaults: dict[str, (bool or dict or dict[str, dict[str, str]] or str or list[str] or list[list]) | None] = {
        "MACRO": list[str],
        "MACRO_C": dict,
        "HELPER": dict,
        "debug": str,
        "id": str,
        "st-load": False,
        "comm-his": list[list],
        "develop-mode": bool,
    }

    cluster_manager: ClusterManager
    root_blob_storage: BlobStorage
    config_fh: FileHandler
    _debug: bool
    flows: dict[str, Callable]
    dev_modi: bool
    functions: dict[str, Any]
    modules: dict[str, Any]

    interface_type: ToolBoxInterfaces
    REFIX: str

    alive: bool
    called_exit: tuple[bool, float]
    args_sto: AppArgs
    system_flag = None
    session = None
    appdata = None
    exit_tasks = []

    enable_profiling: bool = False
    sto = None

    websocket_handlers: dict[str, dict[str, Callable]] = {}
    _rust_ws_bridge: Any = None

    docs_reader: Callable | None = None
    docs_writer: Callable | None = None
    get_update_suggestions: Callable | None = None
    auto_update_docs: Callable | None = None
    source_code_lookup: Callable | None = None

    initial_docs_parse: Callable | None = None

    def __init__(self, prefix=None, args=None):
        self.args_sto = args
        self.prefix = prefix
        self._footprint_start_time = time.time()
        self._process = psutil.Process(os.getpid())

        # Tracking-Daten für Min/Max/Avg
        self._footprint_metrics = {
            'memory': {'max': 0, 'min': float('inf'), 'samples': []},
            'cpu': {'max': 0, 'min': float('inf'), 'samples': []},
            'disk_read': {'max': 0, 'min': float('inf'), 'samples': []},
            'disk_write': {'max': 0, 'min': float('inf'), 'samples': []},
            'network_sent': {'max': 0, 'min': float('inf'), 'samples': []},
            'network_recv': {'max': 0, 'min': float('inf'), 'samples': []},
        }

        # Initial Disk/Network Counters
        try:
            io_counters = self._process.io_counters()
            self._initial_disk_read = io_counters.read_bytes
            self._initial_disk_write = io_counters.write_bytes
        except (AttributeError, OSError):
            self._initial_disk_read = 0
            self._initial_disk_write = 0

        try:
            net_io = psutil.net_io_counters()
            self._initial_network_sent = net_io.bytes_sent
            self._initial_network_recv = net_io.bytes_recv
        except (AttributeError, OSError):
            self._initial_network_sent = 0
            self._initial_network_recv = 0

    def _update_metric_tracking(self, metric_name: str, value: float):
        """Aktualisiert Min/Max/Avg für eine Metrik"""
        metrics = self._footprint_metrics[metric_name]
        metrics['max'] = max(metrics['max'], value)
        metrics['min'] = min(metrics['min'], value)
        metrics['samples'].append(value)

        # Begrenze die Anzahl der Samples (letzte 1000)
        if len(metrics['samples']) > 1000:
            metrics['samples'] = metrics['samples'][-1000:]

    def _get_metric_avg(self, metric_name: str) -> float:
        """Berechnet Durchschnitt einer Metrik"""
        samples = self._footprint_metrics[metric_name]['samples']
        return sum(samples) / len(samples) if samples else 0

    def footprint(self, update_tracking: bool = True) -> FootprintMetrics:
        """
        Erfasst den aktuellen Ressourcen-Footprint der Toolbox-Instanz.

        Args:
            update_tracking: Wenn True, aktualisiert Min/Max/Avg-Tracking

        Returns:
            FootprintMetrics mit allen erfassten Metriken
        """
        current_time = time.time()
        uptime_seconds = current_time - self._footprint_start_time

        # Formatierte Uptime
        uptime_delta = timedelta(seconds=int(uptime_seconds))
        uptime_formatted = str(uptime_delta)

        # Memory Metrics (in MB)
        try:
            mem_info = self._process.memory_info()
            memory_current = mem_info.rss / (1024 * 1024)  # Bytes zu MB
            memory_percent = self._process.memory_percent()

            if update_tracking:
                self._update_metric_tracking('memory', memory_current)

            memory_max = self._footprint_metrics['memory']['max']
            memory_min = self._footprint_metrics['memory']['min']
            if memory_min == float('inf'):
                memory_min = memory_current
        except (psutil.NoSuchProcess, psutil.AccessDenied):
            memory_current = memory_max = memory_min = memory_percent = 0

        # CPU Metrics
        try:
            cpu_percent_current = self._process.cpu_percent(interval=0.1)
            cpu_times = self._process.cpu_times()
            cpu_time_seconds = cpu_times.user + cpu_times.system

            if update_tracking:
                self._update_metric_tracking('cpu', cpu_percent_current)

            cpu_percent_max = self._footprint_metrics['cpu']['max']
            cpu_percent_min = self._footprint_metrics['cpu']['min']
            cpu_percent_avg = self._get_metric_avg('cpu')

            if cpu_percent_min == float('inf'):
                cpu_percent_min = cpu_percent_current
        except (psutil.NoSuchProcess, psutil.AccessDenied):
            cpu_percent_current = cpu_percent_max = 0
            cpu_percent_min = cpu_percent_avg = cpu_time_seconds = 0

        # Disk I/O Metrics (in MB)
        try:
            io_counters = self._process.io_counters()
            disk_read_bytes = io_counters.read_bytes - self._initial_disk_read
            disk_write_bytes = io_counters.write_bytes - self._initial_disk_write

            disk_read_mb = disk_read_bytes / (1024 * 1024)
            disk_write_mb = disk_write_bytes / (1024 * 1024)

            if update_tracking:
                self._update_metric_tracking('disk_read', disk_read_mb)
                self._update_metric_tracking('disk_write', disk_write_mb)

            disk_read_max = self._footprint_metrics['disk_read']['max']
            disk_read_min = self._footprint_metrics['disk_read']['min']
            disk_write_max = self._footprint_metrics['disk_write']['max']
            disk_write_min = self._footprint_metrics['disk_write']['min']

            if disk_read_min == float('inf'):
                disk_read_min = disk_read_mb
            if disk_write_min == float('inf'):
                disk_write_min = disk_write_mb
        except (AttributeError, OSError, psutil.NoSuchProcess, psutil.AccessDenied):
            disk_read_mb = disk_write_mb = 0
            disk_read_max = disk_read_min = disk_write_max = disk_write_min = 0

        # Network I/O Metrics (in MB)
        try:
            net_io = psutil.net_io_counters()
            network_sent_bytes = net_io.bytes_sent - self._initial_network_sent
            network_recv_bytes = net_io.bytes_recv - self._initial_network_recv

            network_sent_mb = network_sent_bytes / (1024 * 1024)
            network_recv_mb = network_recv_bytes / (1024 * 1024)

            if update_tracking:
                self._update_metric_tracking('network_sent', network_sent_mb)
                self._update_metric_tracking('network_recv', network_recv_mb)

            network_sent_max = self._footprint_metrics['network_sent']['max']
            network_sent_min = self._footprint_metrics['network_sent']['min']
            network_recv_max = self._footprint_metrics['network_recv']['max']
            network_recv_min = self._footprint_metrics['network_recv']['min']

            if network_sent_min == float('inf'):
                network_sent_min = network_sent_mb
            if network_recv_min == float('inf'):
                network_recv_min = network_recv_mb
        except (AttributeError, OSError):
            network_sent_mb = network_recv_mb = 0
            network_sent_max = network_sent_min = 0
            network_recv_max = network_recv_min = 0

        # Process Info
        try:
            process_id = self._process.pid
            threads = self._process.num_threads()
            open_files_path = [str(x.path).replace("\\", "/") for x in self._process.open_files()]
            connections_uri = [f"{x.laddr}:{x.raddr} {str(x.status)}" for x in self._process.connections()]

            open_files = len(open_files_path)
            connections = len(connections_uri)
        except (psutil.NoSuchProcess, psutil.AccessDenied):
            process_id = os.getpid()
            threads = open_files = connections = 0
            open_files_path = []
            connections_uri = []

        return FootprintMetrics(
            start_time=self._footprint_start_time,
            uptime_seconds=uptime_seconds,
            uptime_formatted=uptime_formatted,
            memory_current=memory_current,
            memory_max=memory_max,
            memory_min=memory_min,
            memory_percent=memory_percent,
            cpu_percent_current=cpu_percent_current,
            cpu_percent_max=cpu_percent_max,
            cpu_percent_min=cpu_percent_min,
            cpu_percent_avg=cpu_percent_avg,
            cpu_time_seconds=cpu_time_seconds,
            disk_read_mb=disk_read_mb,
            disk_write_mb=disk_write_mb,
            disk_read_max=disk_read_max,
            disk_read_min=disk_read_min,
            disk_write_max=disk_write_max,
            disk_write_min=disk_write_min,
            network_sent_mb=network_sent_mb,
            network_recv_mb=network_recv_mb,
            network_sent_max=network_sent_max,
            network_sent_min=network_sent_min,
            network_recv_max=network_recv_max,
            network_recv_min=network_recv_min,
            process_id=process_id,
            threads=threads,
            open_files=open_files,
            connections=connections,
            open_files_path=open_files_path,
            connections_uri=connections_uri,
        )

    def print_footprint(self, detailed: bool = True) -> str:
        """
        Gibt den Footprint formatiert aus.

        Args:
            detailed: Wenn True, zeigt alle Details, sonst nur Zusammenfassung

        Returns:
            Formatierter Footprint-String
        """
        metrics = self.footprint()

        output = [
            "=" * 70,
            f"TOOLBOX FOOTPRINT - {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}",
            "=" * 70,
            f"\n📊 UPTIME",
            f"  Runtime: {metrics.uptime_formatted}",
            f"  Seconds: {metrics.uptime_seconds:.2f}s",
            f"\n💾 MEMORY USAGE",
            f"  Current:  {metrics.memory_current:.2f} MB ({metrics.memory_percent:.2f}%)",
            f"  Maximum:  {metrics.memory_max:.2f} MB",
            f"  Minimum:  {metrics.memory_min:.2f} MB",
        ]

        if detailed:
            helper_ = '\n\t- '.join(metrics.open_files_path)
            helper__ = '\n\t- '.join(metrics.connections_uri)
            output.extend([
                f"\n⚙️  CPU USAGE",
                f"  Current:  {metrics.cpu_percent_current:.2f}%",
                f"  Maximum:  {metrics.cpu_percent_max:.2f}%",
                f"  Minimum:  {metrics.cpu_percent_min:.2f}%",
                f"  Average:  {metrics.cpu_percent_avg:.2f}%",
                f"  CPU Time: {metrics.cpu_time_seconds:.2f}s",
                f"\n💿 DISK I/O",
                f"  Read:     {metrics.disk_read_mb:.2f} MB (Max: {metrics.disk_read_max:.2f}, Min: {metrics.disk_read_min:.2f})",
                f"  Write:    {metrics.disk_write_mb:.2f} MB (Max: {metrics.disk_write_max:.2f}, Min: {metrics.disk_write_min:.2f})",
                f"\n🌐 NETWORK I/O",
                f"  Sent:     {metrics.network_sent_mb:.2f} MB (Max: {metrics.network_sent_max:.2f}, Min: {metrics.network_sent_min:.2f})",
                f"  Received: {metrics.network_recv_mb:.2f} MB (Max: {metrics.network_recv_max:.2f}, Min: {metrics.network_recv_min:.2f})",
                f"\n🔧 PROCESS INFO",
                f"  PID:         {metrics.process_id}",
                f"  Threads:     {metrics.threads}",
                f"\n📂 OPEN FILES",
                f"  Open Files:  {metrics.open_files}",
                f"  Open Files Path: \n\t- {helper_}",
                f"\n🔗 NETWORK CONNECTIONS",
                f"  Connections: {metrics.connections}",
                f"  Connections URI: \n\t- {helper__}",
            ])

        output.append("=" * 70)

        return "\n".join(output)



    def start_server(self):
        from toolboxv2.utils.clis.api import manage_server
        if self.is_server:
            return
        manage_server("start")
        self.is_server = False

    @staticmethod
    def exit_main(*args, **kwargs):
        """proxi attr"""

    @staticmethod
    async def hide_console(*args, **kwargs):
        """proxi attr"""

    @staticmethod
    async def show_console(*args, **kwargs):
        """proxi attr"""

    @staticmethod
    async def disconnect(*args, **kwargs):
        """proxi attr"""

    def set_logger(self, debug=False):
        """proxi attr"""

    @property
    def debug(self):
        """proxi attr"""
        return self._debug

    def debug_rains(self, e):
        """proxi attr"""

    def set_flows(self, r):
        """proxi attr"""

    async def run_flows(self, name, **kwargs):
        """proxi attr"""

    def rrun_flows(self, name, **kwargs):
        """proxi attr"""

    def idle(self):
        import time
        self.print("idle")
        try:
            while self.alive:
                time.sleep(1)
        except KeyboardInterrupt:
            pass
        self.print("idle done")

    async def a_idle(self):
        self.print("a idle (running :"+("online)" if hasattr(self, 'daemon_app') else "offline)"))
        try:
            if hasattr(self, 'daemon_app'):
                await self.daemon_app.connect(self)
            else:
                while self.alive:
                    await asyncio.sleep(1)
        except KeyboardInterrupt:
            pass
        self.print("a idle done")

    @debug.setter
    def debug(self, value):
        """proxi attr"""

    def _coppy_mod(self, content, new_mod_dir, mod_name, file_type='py'):
        """proxi attr"""

    def _pre_lib_mod(self, mod_name, path_to="./runtime", file_type='py'):
        """proxi attr"""

    def _copy_load(self, mod_name, file_type='py', **kwargs):
        """proxi attr"""

    def inplace_load_instance(self, mod_name, loc="toolboxv2.mods.", spec='app', save=True):
        """proxi attr"""

    def save_instance(self, instance, modular_id, spec='app', instance_type="file/application", tools_class=None):
        """proxi attr"""

    def save_initialized_module(self, tools_class, spec):
        """proxi attr"""

    def mod_online(self, mod_name, installed=False):
        """proxi attr"""

    def _get_function(self,
                      name: Enum or None,
                      state: bool = True,
                      specification: str = "app",
                      metadata=False, as_str: tuple or None = None, r=0):
        """proxi attr"""

    def save_exit(self):
        """proxi attr"""

    def load_mod(self, mod_name: str, mlm='I', **kwargs):
        """proxi attr"""

    async def init_module(self, modular):
        return await self.load_mod(modular)

    async def load_external_mods(self):
        """proxi attr"""

    async def load_all_mods_in_file(self, working_dir="mods"):
        """proxi attr"""

    def get_all_mods(self, working_dir="mods", path_to="./runtime"):
        """proxi attr"""

    def remove_all_modules(self, delete=False):
        for mod in list(self.functions.keys()):
            self.logger.info(f"closing: {mod}")
            self.remove_mod(mod, delete=delete)

    async def a_remove_all_modules(self, delete=False):
        for mod in list(self.functions.keys()):
            self.logger.info(f"closing: {mod}")
            await self.a_remove_mod(mod, delete=delete)

    def print_ok(self):
        """proxi attr"""
        self.logger.info("OK")

    def reload_mod(self, mod_name, spec='app', is_file=True, loc="toolboxv2.mods."):
        """proxi attr"""

    def watch_mod(self, mod_name, spec='app', loc="toolboxv2.mods.", use_thread=True, path_name=None):
        """proxi attr"""

    def remove_mod(self, mod_name, spec='app', delete=True):
        """proxi attr"""

    async def a_remove_mod(self, mod_name, spec='app', delete=True):
        """proxi attr"""

    def exit(self):
        """proxi attr"""

    def web_context(self) -> str:
        """returns the build index ( toolbox web component )"""

    async def a_exit(self):
        """proxi attr"""

    def save_load(self, modname, spec='app'):
        """proxi attr"""

    def get_function(self, name: Enum or tuple, **kwargs):
        """
        Kwargs for _get_function
            metadata:: return the registered function dictionary
                stateless: (function_data, None), 0
                stateful: (function_data, higher_order_function), 0
            state::boolean
                specification::str default app
        """

    def run_a_from_sync(self, function, *args):
        """
        run a async fuction
        """

    def run_bg_task_advanced(self, task, *args, **kwargs):
        """
        proxi attr
        """

    def wait_for_bg_tasks(self, timeout=None):
        """
        proxi attr
        """

    def run_bg_task(self, task):
        """
                run a async fuction
                """
    def run_function(self, mod_function_name: Enum or tuple,
                     tb_run_function_with_state=True,
                     tb_run_with_specification='app',
                     args_=None,
                     kwargs_=None,
                     *args,
                     **kwargs) -> Result:

        """proxi attr"""

    async def a_run_function(self, mod_function_name: Enum or tuple,
                             tb_run_function_with_state=True,
                             tb_run_with_specification='app',
                             args_=None,
                             kwargs_=None,
                             *args,
                             **kwargs) -> Result:

        """proxi attr"""

    def fuction_runner(self, function, function_data: dict, args: list, kwargs: dict, t0=.0):
        """
        parameters = function_data.get('params')
        modular_name = function_data.get('module_name')
        function_name = function_data.get('func_name')
        mod_function_name = f"{modular_name}.{function_name}"

        proxi attr
        """

    async def a_fuction_runner(self, function, function_data: dict, args: list, kwargs: dict):
        """
        parameters = function_data.get('params')
        modular_name = function_data.get('module_name')
        function_name = function_data.get('func_name')
        mod_function_name = f"{modular_name}.{function_name}"

        proxi attr
        """

    async def run_http(self, mod_function_name: Enum or str or tuple, function_name=None, method="GET",
                       args_=None,
                       kwargs_=None,
                       *args, **kwargs):
        """run a function remote via http / https"""

    def run_any(self, mod_function_name: Enum or str or tuple, backwords_compability_variabel_string_holder=None,
                get_results=False, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None,
                kwargs_=None,
                *args, **kwargs):
        """proxi attr"""

    async def a_run_any(self, mod_function_name: Enum or str or tuple,
                        backwords_compability_variabel_string_holder=None,
                        get_results=False, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None,
                        kwargs_=None,
                        *args, **kwargs):
        """proxi attr"""

    def get_mod(self, name, spec='app') -> ModuleType or MainToolType:
        """proxi attr"""

    @staticmethod
    def print(text, *args, **kwargs):
        """proxi attr"""

    @staticmethod
    def sprint(text, *args, **kwargs):
        """proxi attr"""

    # ----------------------------------------------------------------
    # Decorators for the toolbox

    def _register_function(self, module_name, func_name, data):
        """proxi attr"""

    def _create_decorator(self, type_: str,
                          name: str = "",
                          mod_name: str = "",
                          level: int = -1,
                          restrict_in_virtual_mode: bool = False,
                          api: bool = False,
                          helper: str = "",
                          version: str or None = None,
                          initial=False,
                          exit_f=False,
                          test=True,
                          samples=None,
                          state=None,
                          pre_compute=None,
                          post_compute=None,
                          memory_cache=False,
                          file_cache=False,
                          row=False,
                          request_as_kwarg=False,
                          memory_cache_max_size=100,
                          memory_cache_ttl=300,
                          websocket_handler: str | None = None,):
        """proxi attr"""

        # data = {
        #     "type": type_,
        #     "module_name": module_name,
        #     "func_name": func_name,
        #     "level": level,
        #     "restrict_in_virtual_mode": restrict_in_virtual_mode,
        #     "func": func,
        #     "api": api,
        #     "helper": helper,
        #     "version": version,
        #     "initial": initial,
        #     "exit_f": exit_f,
        #     "__module__": func.__module__,
        #     "signature": sig,
        #     "params": params,
        #     "state": (
        #         False if len(params) == 0 else params[0] in ['self', 'state', 'app']) if state is None else state,
        #     "do_test": test,
        #     "samples": samples,
        #     "request_as_kwarg": request_as_kwarg,

    def tb(self, name=None,
           mod_name: str = "",
           helper: str = "",
           version: str or None = None,
           test: bool = True,
           restrict_in_virtual_mode: bool = False,
           api: bool = False,
           initial: bool = False,
           exit_f: bool = False,
           test_only: bool = False,
           memory_cache: bool = False,
           file_cache: bool = False,
           row=False,
           request_as_kwarg: bool = False,
           state: bool or None = None,
           level: int = 0,
           memory_cache_max_size: int = 100,
           memory_cache_ttl: int = 300,
           samples: list or dict or None = None,
           interface: ToolBoxInterfaces or None or str = None,
           pre_compute=None,
           post_compute=None,
           api_methods=None,
           websocket_handler: str | None = None,
           ):
        """
    A decorator for registering and configuring functions within a module.

    This decorator is used to wrap functions with additional functionality such as caching, API conversion, and lifecycle management (initialization and exit). It also handles the registration of the function in the module's function registry.

    Args:
        name (str, optional): The name to register the function under. Defaults to the function's own name.
        mod_name (str, optional): The name of the module the function belongs to.
        helper (str, optional): A helper string providing additional information about the function.
        version (str or None, optional): The version of the function or module.
        test (bool, optional): Flag to indicate if the function is for testing purposes.
        restrict_in_virtual_mode (bool, optional): Flag to restrict the function in virtual mode.
        api (bool, optional): Flag to indicate if the function is part of an API.
        initial (bool, optional): Flag to indicate if the function should be executed at initialization.
        exit_f (bool, optional): Flag to indicate if the function should be executed at exit.
        test_only (bool, optional): Flag to indicate if the function should only be used for testing.
        memory_cache (bool, optional): Flag to enable memory caching for the function.
        request_as_kwarg (bool, optional): Flag to get request if the fuction is calld from api.
        file_cache (bool, optional): Flag to enable file caching for the function.
        row (bool, optional): rather to auto wrap the result in Result type default False means no row data aka result type
        state (bool or None, optional): Flag to indicate if the function maintains state.
        level (int, optional): The level of the function, used for prioritization or categorization.
        memory_cache_max_size (int, optional): Maximum size of the memory cache.
        memory_cache_ttl (int, optional): Time-to-live for the memory cache entries.
        samples (list or dict or None, optional): Samples or examples of function usage.
        interface (str, optional): The interface type for the function.
        pre_compute (callable, optional): A function to be called before the main function.
        post_compute (callable, optional): A function to be called after the main function.
        api_methods (list[str], optional): default ["AUTO"] (GET if not params, POST if params) , GET, POST, PUT or DELETE.

    Returns:
        function: The decorated function with additional processing and registration capabilities.
    """
        if interface is None:
            interface = "tb"
        if test_only and 'test' not in self.id:
            return lambda *args, **kwargs: args
        return self._create_decorator(interface,
                                      name,
                                      mod_name,
                                      level=level,
                                      restrict_in_virtual_mode=restrict_in_virtual_mode,
                                      helper=helper,
                                      api=api,
                                      version=version,
                                      initial=initial,
                                      exit_f=exit_f,
                                      test=test,
                                      samples=samples,
                                      state=state,
                                      pre_compute=pre_compute,
                                      post_compute=post_compute,
                                      memory_cache=memory_cache,
                                      file_cache=file_cache,
                                      row=row,
                                      request_as_kwarg=request_as_kwarg,
                                      memory_cache_max_size=memory_cache_max_size,
                                      memory_cache_ttl=memory_cache_ttl)

    def print_functions(self, name=None):


        if not self.functions:
            return

        def helper(_functions):
            for func_name, data in _functions.items():
                if not isinstance(data, dict):
                    continue

                func_type = data.get('type', 'Unknown')
                func_level = 'r' if data['level'] == -1 else data['level']
                api_status = 'Api' if data.get('api', False) else 'Non-Api'

                print(f"  Function: {func_name}{data.get('signature', '()')}; "
                      f"Type: {func_type}, Level: {func_level}, {api_status}")

        if name is not None:
            functions = self.functions.get(name)
            if functions is not None:
                print(f"\nModule: {name}; Type: {functions.get('app_instance_type', 'Unknown')}")
                helper(functions)
                return
        for module, functions in self.functions.items():
            print(f"\nModule: {module}; Type: {functions.get('app_instance_type', 'Unknown')}")
            helper(functions)

    def save_autocompletion_dict(self):
        """proxi attr"""

    def get_autocompletion_dict(self):
        """proxi attr"""

    def get_username(self, get_input=False, default="loot") -> str:
        """proxi attr"""

    def save_registry_as_enums(self, directory: str, filename: str):
        """proxi attr"""

    async def execute_all_functions_(self, m_query='', f_query=''):
        print("Executing all functions")
        from ..extras import generate_test_cases
        all_data = {
            "modular_run": 0,
            "modular_fatal_error": 0,
            "errors": 0,
            "modular_sug": 0,
            "coverage": [],
            "total_coverage": {},
        }
        items = list(self.functions.items()).copy()
        for module_name, functions in items:
            infos = {
                "functions_run": 0,
                "functions_fatal_error": 0,
                "error": 0,
                "functions_sug": 0,
                'calls': {},
                'callse': {},
                "coverage": [0, 0],
            }
            all_data['modular_run'] += 1
            if not module_name.startswith(m_query):
                all_data['modular_sug'] += 1
                continue

            with Spinner(message=f"In {module_name}| "):
                f_items = list(functions.items()).copy()
                for function_name, function_data in f_items:
                    if not isinstance(function_data, dict):
                        continue
                    if not function_name.startswith(f_query):
                        continue
                    test: list = function_data.get('do_test')
                    # print(test, module_name, function_name, function_data)
                    infos["coverage"][0] += 1
                    if test is False:
                        continue

                    with Spinner(message=f"\t\t\t\t\t\tfuction {function_name}..."):
                        params: list = function_data.get('params')
                        sig: signature = function_data.get('signature')
                        state: bool = function_data.get('state')
                        samples: bool = function_data.get('samples')

                        test_kwargs_list = [{}]

                        if params is not None:
                            test_kwargs_list = samples if samples is not None else generate_test_cases(sig=sig)
                            # print(test_kwargs)
                            # print(test_kwargs[0])
                            # test_kwargs = test_kwargs_list[0]
                        # print(module_name, function_name, test_kwargs_list)
                        infos["coverage"][1] += 1
                        for test_kwargs in test_kwargs_list:
                            try:
                                # print(f"test Running {state=} |{module_name}.{function_name}")
                                result = await self.a_run_function((module_name, function_name),
                                                                   tb_run_function_with_state=state,
                                                                   **test_kwargs)
                                if not isinstance(result, Result):
                                    result = Result.ok(result)
                                if result.info.exec_code == 0:
                                    infos['calls'][function_name] = [test_kwargs, str(result)]
                                    infos['functions_sug'] += 1
                                else:
                                    infos['functions_sug'] += 1
                                    infos['error'] += 1
                                    infos['callse'][function_name] = [test_kwargs, str(result)]
                            except Exception as e:
                                infos['functions_fatal_error'] += 1
                                infos['callse'][function_name] = [test_kwargs, str(e)]
                            finally:
                                infos['functions_run'] += 1

                if infos['functions_run'] == infos['functions_sug']:
                    all_data['modular_sug'] += 1
                else:
                    all_data['modular_fatal_error'] += 1
                if infos['error'] > 0:
                    all_data['errors'] += infos['error']

                all_data[module_name] = infos
                if infos['coverage'][0] == 0:
                    c = 0
                else:
                    c = infos['coverage'][1] / infos['coverage'][0]
                all_data["coverage"].append(f"{module_name}:{c:.2f}\n")
        total_coverage = sum([float(t.split(":")[-1]) for t in all_data["coverage"]]) / len(all_data["coverage"])
        print(
            f"\n{all_data['modular_run']=}\n{all_data['modular_sug']=}\n{all_data['modular_fatal_error']=}\n{total_coverage=}")
        d = analyze_data(all_data)
        return Result.ok(data=all_data, data_info=d)

    async def execute_function_test(self, module_name: str, function_name: str,
                                    function_data: dict, test_kwargs: dict,
                                    profiler: cProfile.Profile) -> tuple[bool, str, dict, float]:
        start_time = time.time()
        with profile_section(profiler, hasattr(self, 'enable_profiling') and self.enable_profiling):
            try:
                result = await self.a_run_function(
                    (module_name, function_name),
                    tb_run_function_with_state=function_data.get('state'),
                    **test_kwargs
                )

                if not isinstance(result, Result):
                    result = Result.ok(result)

                success = result.info.exec_code == 0
                execution_time = time.time() - start_time
                return success, str(result), test_kwargs, execution_time
            except Exception as e:
                execution_time = time.time() - start_time
                return False, str(e), test_kwargs, execution_time

    async def process_function(self, module_name: str, function_name: str,
                               function_data: dict, profiler: cProfile.Profile) -> tuple[str, ModuleInfo]:
        start_time = time.time()
        info = ModuleInfo()

        with profile_section(profiler, hasattr(self, 'enable_profiling') and self.enable_profiling):
            if not isinstance(function_data, dict):
                return function_name, info

            test = function_data.get('do_test')
            info.coverage[0] += 1

            if test is False:
                return function_name, info

            params = function_data.get('params')
            sig = function_data.get('signature')
            samples = function_data.get('samples')

            test_kwargs_list = [{}] if params is None else (
                samples if samples is not None else generate_test_cases(sig=sig)
            )

            info.coverage[1] += 1

            # Create tasks for all test cases
            tasks = [
                self.execute_function_test(module_name, function_name, function_data, test_kwargs, profiler)
                for test_kwargs in test_kwargs_list
            ]

            # Execute all tests concurrently
            results = await asyncio.gather(*tasks)

            total_execution_time = 0
            for success, result_str, test_kwargs, execution_time in results:
                info.functions_run += 1
                total_execution_time += execution_time

                if success:
                    info.functions_sug += 1
                    info.calls[function_name] = [test_kwargs, result_str]
                else:
                    info.functions_sug += 1
                    info.error += 1
                    info.callse[function_name] = [test_kwargs, result_str]

            info.execution_time = time.time() - start_time
            return function_name, info

    async def process_module(self, module_name: str, functions: dict,
                             f_query: str, profiler: cProfile.Profile) -> tuple[str, ModuleInfo]:
        start_time = time.time()

        with profile_section(profiler, hasattr(self, 'enable_profiling') and self.enable_profiling):
            async with asyncio.Semaphore(mp.cpu_count()):
                tasks = [
                    self.process_function(module_name, fname, fdata, profiler)
                    for fname, fdata in functions.items()
                    if fname.startswith(f_query)
                ]

                if not tasks:
                    return module_name, ModuleInfo()

                results = await asyncio.gather(*tasks)

                # Combine results from all functions in the module
                combined_info = ModuleInfo()
                total_execution_time = 0

                for _, info in results:
                    combined_info.functions_run += info.functions_run
                    combined_info.functions_fatal_error += info.functions_fatal_error
                    combined_info.error += info.error
                    combined_info.functions_sug += info.functions_sug
                    combined_info.calls.update(info.calls)
                    combined_info.callse.update(info.callse)
                    combined_info.coverage[0] += info.coverage[0]
                    combined_info.coverage[1] += info.coverage[1]
                    total_execution_time += info.execution_time

                combined_info.execution_time = time.time() - start_time
                return module_name, combined_info

    async def execute_all_functions(self, m_query='', f_query='', enable_profiling=True):
        """
        Execute all functions with parallel processing and optional profiling.

        Args:
            m_query (str): Module name query filter
            f_query (str): Function name query filter
            enable_profiling (bool): Enable detailed profiling information
        """
        print("Executing all functions in parallel" + (" with profiling" if enable_profiling else ""))

        start_time = time.time()
        stats = ExecutionStats()
        items = list(self.functions.items()).copy()

        # Set up profiling
        self.enable_profiling = enable_profiling
        profiler = cProfile.Profile()

        with profile_section(profiler, enable_profiling):
            # Filter modules based on query
            filtered_modules = [
                (mname, mfuncs) for mname, mfuncs in items
                if mname.startswith(m_query)
            ]

            stats.modular_run = len(filtered_modules)

            # Process all modules concurrently
            async with asyncio.Semaphore(mp.cpu_count()):
                tasks = [
                    self.process_module(mname, mfuncs, f_query, profiler)
                    for mname, mfuncs in filtered_modules
                ]

                results = await asyncio.gather(*tasks)

            # Combine results and calculate statistics
            for module_name, info in results:
                if info.functions_run == info.functions_sug:
                    stats.modular_sug += 1
                else:
                    stats.modular_fatal_error += 1

                stats.errors += info.error

                # Calculate coverage
                coverage = (info.coverage[1] / info.coverage[0]) if info.coverage[0] > 0 else 0
                stats.coverage.append(f"{module_name}:{coverage:.2f}\n")

                # Store module info
                stats.__dict__[module_name] = info

            # Calculate total coverage
            total_coverage = (
                sum(float(t.split(":")[-1]) for t in stats.coverage) / len(stats.coverage)
                if stats.coverage else 0
            )

            stats.total_execution_time = time.time() - start_time

            # Generate profiling stats if enabled
            if enable_profiling:
                s = io.StringIO()
                ps = pstats.Stats(profiler, stream=s).sort_stats('cumulative')
                ps.print_stats()
                stats.profiling_data = {
                    'detailed_stats': s.getvalue(),
                    'total_time': stats.total_execution_time,
                    'function_count': stats.modular_run,
                    'successful_functions': stats.modular_sug
                }

            print(
                f"\n{stats.modular_run=}"
                f"\n{stats.modular_sug=}"
                f"\n{stats.modular_fatal_error=}"
                f"\n{total_coverage=}"
                f"\nTotal execution time: {stats.total_execution_time:.2f}s"
            )

            if enable_profiling:
                print("\nProfiling Summary:")
                print(f"{'=' * 50}")
                print("Top 10 time-consuming functions:")
                ps.print_stats(10)

            analyzed_data = analyze_data(stats.__dict__)
            return Result.ok(data=stats.__dict__, data_info=analyzed_data)

debug property writable

proxi attr

a_exit() async

proxi attr

Source code in toolboxv2/utils/system/types.py

async def a_exit(self):
    """proxi attr"""

a_fuction_runner(function, function_data, args, kwargs) async

parameters = function_data.get('params') modular_name = function_data.get('module_name') function_name = function_data.get('func_name') mod_function_name = f"{modular_name}.{function_name}"

proxi attr

Source code in toolboxv2/utils/system/types.py

async def a_fuction_runner(self, function, function_data: dict, args: list, kwargs: dict):
    """
    parameters = function_data.get('params')
    modular_name = function_data.get('module_name')
    function_name = function_data.get('func_name')
    mod_function_name = f"{modular_name}.{function_name}"

    proxi attr
    """

a_remove_mod(mod_name, spec='app', delete=True) async

proxi attr

Source code in toolboxv2/utils/system/types.py

async def a_remove_mod(self, mod_name, spec='app', delete=True):
    """proxi attr"""

a_run_any(mod_function_name, backwords_compability_variabel_string_holder=None, get_results=False, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None, kwargs_=None, *args, **kwargs) async

proxi attr

Source code in toolboxv2/utils/system/types.py

async def a_run_any(self, mod_function_name: Enum or str or tuple,
                    backwords_compability_variabel_string_holder=None,
                    get_results=False, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None,
                    kwargs_=None,
                    *args, **kwargs):
    """proxi attr"""

a_run_function(mod_function_name, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None, kwargs_=None, *args, **kwargs) async

proxi attr

Source code in toolboxv2/utils/system/types.py

async def a_run_function(self, mod_function_name: Enum or tuple,
                         tb_run_function_with_state=True,
                         tb_run_with_specification='app',
                         args_=None,
                         kwargs_=None,
                         *args,
                         **kwargs) -> Result:

    """proxi attr"""

debug_rains(e)

proxi attr

Source code in toolboxv2/utils/system/types.py

def debug_rains(self, e):
    """proxi attr"""

disconnect(*args, **kwargs) async staticmethod

proxi attr

Source code in toolboxv2/utils/system/types.py

@staticmethod
async def disconnect(*args, **kwargs):
    """proxi attr"""

execute_all_functions(m_query='', f_query='', enable_profiling=True) async

Execute all functions with parallel processing and optional profiling.

Parameters:

Name	Type	Description	Default
m_query	str	Module name query filter	''
f_query	str	Function name query filter	''
enable_profiling	bool	Enable detailed profiling information	True

Source code in toolboxv2/utils/system/types.py

async def execute_all_functions(self, m_query='', f_query='', enable_profiling=True):
    """
    Execute all functions with parallel processing and optional profiling.

    Args:
        m_query (str): Module name query filter
        f_query (str): Function name query filter
        enable_profiling (bool): Enable detailed profiling information
    """
    print("Executing all functions in parallel" + (" with profiling" if enable_profiling else ""))

    start_time = time.time()
    stats = ExecutionStats()
    items = list(self.functions.items()).copy()

    # Set up profiling
    self.enable_profiling = enable_profiling
    profiler = cProfile.Profile()

    with profile_section(profiler, enable_profiling):
        # Filter modules based on query
        filtered_modules = [
            (mname, mfuncs) for mname, mfuncs in items
            if mname.startswith(m_query)
        ]

        stats.modular_run = len(filtered_modules)

        # Process all modules concurrently
        async with asyncio.Semaphore(mp.cpu_count()):
            tasks = [
                self.process_module(mname, mfuncs, f_query, profiler)
                for mname, mfuncs in filtered_modules
            ]

            results = await asyncio.gather(*tasks)

        # Combine results and calculate statistics
        for module_name, info in results:
            if info.functions_run == info.functions_sug:
                stats.modular_sug += 1
            else:
                stats.modular_fatal_error += 1

            stats.errors += info.error

            # Calculate coverage
            coverage = (info.coverage[1] / info.coverage[0]) if info.coverage[0] > 0 else 0
            stats.coverage.append(f"{module_name}:{coverage:.2f}\n")

            # Store module info
            stats.__dict__[module_name] = info

        # Calculate total coverage
        total_coverage = (
            sum(float(t.split(":")[-1]) for t in stats.coverage) / len(stats.coverage)
            if stats.coverage else 0
        )

        stats.total_execution_time = time.time() - start_time

        # Generate profiling stats if enabled
        if enable_profiling:
            s = io.StringIO()
            ps = pstats.Stats(profiler, stream=s).sort_stats('cumulative')
            ps.print_stats()
            stats.profiling_data = {
                'detailed_stats': s.getvalue(),
                'total_time': stats.total_execution_time,
                'function_count': stats.modular_run,
                'successful_functions': stats.modular_sug
            }

        print(
            f"\n{stats.modular_run=}"
            f"\n{stats.modular_sug=}"
            f"\n{stats.modular_fatal_error=}"
            f"\n{total_coverage=}"
            f"\nTotal execution time: {stats.total_execution_time:.2f}s"
        )

        if enable_profiling:
            print("\nProfiling Summary:")
            print(f"{'=' * 50}")
            print("Top 10 time-consuming functions:")
            ps.print_stats(10)

        analyzed_data = analyze_data(stats.__dict__)
        return Result.ok(data=stats.__dict__, data_info=analyzed_data)

exit()

proxi attr

Source code in toolboxv2/utils/system/types.py

def exit(self):
    """proxi attr"""

exit_main(*args, **kwargs) staticmethod

proxi attr

Source code in toolboxv2/utils/system/types.py

@staticmethod
def exit_main(*args, **kwargs):
    """proxi attr"""

footprint(update_tracking=True)

Erfasst den aktuellen Ressourcen-Footprint der Toolbox-Instanz.

Parameters:

Name	Type	Description	Default
update_tracking	bool	Wenn True, aktualisiert Min/Max/Avg-Tracking	True

Returns:

Type	Description
FootprintMetrics	FootprintMetrics mit allen erfassten Metriken

Source code in toolboxv2/utils/system/types.py

def footprint(self, update_tracking: bool = True) -> FootprintMetrics:
    """
    Erfasst den aktuellen Ressourcen-Footprint der Toolbox-Instanz.

    Args:
        update_tracking: Wenn True, aktualisiert Min/Max/Avg-Tracking

    Returns:
        FootprintMetrics mit allen erfassten Metriken
    """
    current_time = time.time()
    uptime_seconds = current_time - self._footprint_start_time

    # Formatierte Uptime
    uptime_delta = timedelta(seconds=int(uptime_seconds))
    uptime_formatted = str(uptime_delta)

    # Memory Metrics (in MB)
    try:
        mem_info = self._process.memory_info()
        memory_current = mem_info.rss / (1024 * 1024)  # Bytes zu MB
        memory_percent = self._process.memory_percent()

        if update_tracking:
            self._update_metric_tracking('memory', memory_current)

        memory_max = self._footprint_metrics['memory']['max']
        memory_min = self._footprint_metrics['memory']['min']
        if memory_min == float('inf'):
            memory_min = memory_current
    except (psutil.NoSuchProcess, psutil.AccessDenied):
        memory_current = memory_max = memory_min = memory_percent = 0

    # CPU Metrics
    try:
        cpu_percent_current = self._process.cpu_percent(interval=0.1)
        cpu_times = self._process.cpu_times()
        cpu_time_seconds = cpu_times.user + cpu_times.system

        if update_tracking:
            self._update_metric_tracking('cpu', cpu_percent_current)

        cpu_percent_max = self._footprint_metrics['cpu']['max']
        cpu_percent_min = self._footprint_metrics['cpu']['min']
        cpu_percent_avg = self._get_metric_avg('cpu')

        if cpu_percent_min == float('inf'):
            cpu_percent_min = cpu_percent_current
    except (psutil.NoSuchProcess, psutil.AccessDenied):
        cpu_percent_current = cpu_percent_max = 0
        cpu_percent_min = cpu_percent_avg = cpu_time_seconds = 0

    # Disk I/O Metrics (in MB)
    try:
        io_counters = self._process.io_counters()
        disk_read_bytes = io_counters.read_bytes - self._initial_disk_read
        disk_write_bytes = io_counters.write_bytes - self._initial_disk_write

        disk_read_mb = disk_read_bytes / (1024 * 1024)
        disk_write_mb = disk_write_bytes / (1024 * 1024)

        if update_tracking:
            self._update_metric_tracking('disk_read', disk_read_mb)
            self._update_metric_tracking('disk_write', disk_write_mb)

        disk_read_max = self._footprint_metrics['disk_read']['max']
        disk_read_min = self._footprint_metrics['disk_read']['min']
        disk_write_max = self._footprint_metrics['disk_write']['max']
        disk_write_min = self._footprint_metrics['disk_write']['min']

        if disk_read_min == float('inf'):
            disk_read_min = disk_read_mb
        if disk_write_min == float('inf'):
            disk_write_min = disk_write_mb
    except (AttributeError, OSError, psutil.NoSuchProcess, psutil.AccessDenied):
        disk_read_mb = disk_write_mb = 0
        disk_read_max = disk_read_min = disk_write_max = disk_write_min = 0

    # Network I/O Metrics (in MB)
    try:
        net_io = psutil.net_io_counters()
        network_sent_bytes = net_io.bytes_sent - self._initial_network_sent
        network_recv_bytes = net_io.bytes_recv - self._initial_network_recv

        network_sent_mb = network_sent_bytes / (1024 * 1024)
        network_recv_mb = network_recv_bytes / (1024 * 1024)

        if update_tracking:
            self._update_metric_tracking('network_sent', network_sent_mb)
            self._update_metric_tracking('network_recv', network_recv_mb)

        network_sent_max = self._footprint_metrics['network_sent']['max']
        network_sent_min = self._footprint_metrics['network_sent']['min']
        network_recv_max = self._footprint_metrics['network_recv']['max']
        network_recv_min = self._footprint_metrics['network_recv']['min']

        if network_sent_min == float('inf'):
            network_sent_min = network_sent_mb
        if network_recv_min == float('inf'):
            network_recv_min = network_recv_mb
    except (AttributeError, OSError):
        network_sent_mb = network_recv_mb = 0
        network_sent_max = network_sent_min = 0
        network_recv_max = network_recv_min = 0

    # Process Info
    try:
        process_id = self._process.pid
        threads = self._process.num_threads()
        open_files_path = [str(x.path).replace("\\", "/") for x in self._process.open_files()]
        connections_uri = [f"{x.laddr}:{x.raddr} {str(x.status)}" for x in self._process.connections()]

        open_files = len(open_files_path)
        connections = len(connections_uri)
    except (psutil.NoSuchProcess, psutil.AccessDenied):
        process_id = os.getpid()
        threads = open_files = connections = 0
        open_files_path = []
        connections_uri = []

    return FootprintMetrics(
        start_time=self._footprint_start_time,
        uptime_seconds=uptime_seconds,
        uptime_formatted=uptime_formatted,
        memory_current=memory_current,
        memory_max=memory_max,
        memory_min=memory_min,
        memory_percent=memory_percent,
        cpu_percent_current=cpu_percent_current,
        cpu_percent_max=cpu_percent_max,
        cpu_percent_min=cpu_percent_min,
        cpu_percent_avg=cpu_percent_avg,
        cpu_time_seconds=cpu_time_seconds,
        disk_read_mb=disk_read_mb,
        disk_write_mb=disk_write_mb,
        disk_read_max=disk_read_max,
        disk_read_min=disk_read_min,
        disk_write_max=disk_write_max,
        disk_write_min=disk_write_min,
        network_sent_mb=network_sent_mb,
        network_recv_mb=network_recv_mb,
        network_sent_max=network_sent_max,
        network_sent_min=network_sent_min,
        network_recv_max=network_recv_max,
        network_recv_min=network_recv_min,
        process_id=process_id,
        threads=threads,
        open_files=open_files,
        connections=connections,
        open_files_path=open_files_path,
        connections_uri=connections_uri,
    )

fuction_runner(function, function_data, args, kwargs, t0=0.0)

parameters = function_data.get('params') modular_name = function_data.get('module_name') function_name = function_data.get('func_name') mod_function_name = f"{modular_name}.{function_name}"

proxi attr

Source code in toolboxv2/utils/system/types.py

def fuction_runner(self, function, function_data: dict, args: list, kwargs: dict, t0=.0):
    """
    parameters = function_data.get('params')
    modular_name = function_data.get('module_name')
    function_name = function_data.get('func_name')
    mod_function_name = f"{modular_name}.{function_name}"

    proxi attr
    """

get_all_mods(working_dir='mods', path_to='./runtime')

proxi attr

Source code in toolboxv2/utils/system/types.py

def get_all_mods(self, working_dir="mods", path_to="./runtime"):
    """proxi attr"""

get_autocompletion_dict()

proxi attr

Source code in toolboxv2/utils/system/types.py

def get_autocompletion_dict(self):
    """proxi attr"""

get_function(name, **kwargs)

Kwargs for _get_function metadata:: return the registered function dictionary stateless: (function_data, None), 0 stateful: (function_data, higher_order_function), 0 state::boolean specification::str default app

Source code in toolboxv2/utils/system/types.py

def get_function(self, name: Enum or tuple, **kwargs):
    """
    Kwargs for _get_function
        metadata:: return the registered function dictionary
            stateless: (function_data, None), 0
            stateful: (function_data, higher_order_function), 0
        state::boolean
            specification::str default app
    """

get_mod(name, spec='app')

proxi attr

Source code in toolboxv2/utils/system/types.py

def get_mod(self, name, spec='app') -> ModuleType or MainToolType:
    """proxi attr"""

get_username(get_input=False, default='loot')

proxi attr

Source code in toolboxv2/utils/system/types.py

def get_username(self, get_input=False, default="loot") -> str:
    """proxi attr"""

hide_console(*args, **kwargs) async staticmethod

proxi attr

Source code in toolboxv2/utils/system/types.py

@staticmethod
async def hide_console(*args, **kwargs):
    """proxi attr"""

inplace_load_instance(mod_name, loc='toolboxv2.mods.', spec='app', save=True)

proxi attr

Source code in toolboxv2/utils/system/types.py

def inplace_load_instance(self, mod_name, loc="toolboxv2.mods.", spec='app', save=True):
    """proxi attr"""

load_all_mods_in_file(working_dir='mods') async

proxi attr

Source code in toolboxv2/utils/system/types.py

async def load_all_mods_in_file(self, working_dir="mods"):
    """proxi attr"""

load_external_mods() async

proxi attr

Source code in toolboxv2/utils/system/types.py

async def load_external_mods(self):
    """proxi attr"""

load_mod(mod_name, mlm='I', **kwargs)

proxi attr

Source code in toolboxv2/utils/system/types.py

def load_mod(self, mod_name: str, mlm='I', **kwargs):
    """proxi attr"""

mod_online(mod_name, installed=False)

proxi attr

Source code in toolboxv2/utils/system/types.py

def mod_online(self, mod_name, installed=False):
    """proxi attr"""

print(text, *args, **kwargs) staticmethod

proxi attr

Source code in toolboxv2/utils/system/types.py

@staticmethod
def print(text, *args, **kwargs):
    """proxi attr"""

print_footprint(detailed=True)

Gibt den Footprint formatiert aus.

Parameters:

Name	Type	Description	Default
detailed	bool	Wenn True, zeigt alle Details, sonst nur Zusammenfassung	True

Returns:

Type	Description
str	Formatierter Footprint-String

Source code in toolboxv2/utils/system/types.py

def print_footprint(self, detailed: bool = True) -> str:
    """
    Gibt den Footprint formatiert aus.

    Args:
        detailed: Wenn True, zeigt alle Details, sonst nur Zusammenfassung

    Returns:
        Formatierter Footprint-String
    """
    metrics = self.footprint()

    output = [
        "=" * 70,
        f"TOOLBOX FOOTPRINT - {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}",
        "=" * 70,
        f"\n📊 UPTIME",
        f"  Runtime: {metrics.uptime_formatted}",
        f"  Seconds: {metrics.uptime_seconds:.2f}s",
        f"\n💾 MEMORY USAGE",
        f"  Current:  {metrics.memory_current:.2f} MB ({metrics.memory_percent:.2f}%)",
        f"  Maximum:  {metrics.memory_max:.2f} MB",
        f"  Minimum:  {metrics.memory_min:.2f} MB",
    ]

    if detailed:
        helper_ = '\n\t- '.join(metrics.open_files_path)
        helper__ = '\n\t- '.join(metrics.connections_uri)
        output.extend([
            f"\n⚙️  CPU USAGE",
            f"  Current:  {metrics.cpu_percent_current:.2f}%",
            f"  Maximum:  {metrics.cpu_percent_max:.2f}%",
            f"  Minimum:  {metrics.cpu_percent_min:.2f}%",
            f"  Average:  {metrics.cpu_percent_avg:.2f}%",
            f"  CPU Time: {metrics.cpu_time_seconds:.2f}s",
            f"\n💿 DISK I/O",
            f"  Read:     {metrics.disk_read_mb:.2f} MB (Max: {metrics.disk_read_max:.2f}, Min: {metrics.disk_read_min:.2f})",
            f"  Write:    {metrics.disk_write_mb:.2f} MB (Max: {metrics.disk_write_max:.2f}, Min: {metrics.disk_write_min:.2f})",
            f"\n🌐 NETWORK I/O",
            f"  Sent:     {metrics.network_sent_mb:.2f} MB (Max: {metrics.network_sent_max:.2f}, Min: {metrics.network_sent_min:.2f})",
            f"  Received: {metrics.network_recv_mb:.2f} MB (Max: {metrics.network_recv_max:.2f}, Min: {metrics.network_recv_min:.2f})",
            f"\n🔧 PROCESS INFO",
            f"  PID:         {metrics.process_id}",
            f"  Threads:     {metrics.threads}",
            f"\n📂 OPEN FILES",
            f"  Open Files:  {metrics.open_files}",
            f"  Open Files Path: \n\t- {helper_}",
            f"\n🔗 NETWORK CONNECTIONS",
            f"  Connections: {metrics.connections}",
            f"  Connections URI: \n\t- {helper__}",
        ])

    output.append("=" * 70)

    return "\n".join(output)

print_ok()

proxi attr

Source code in toolboxv2/utils/system/types.py

def print_ok(self):
    """proxi attr"""
    self.logger.info("OK")

reload_mod(mod_name, spec='app', is_file=True, loc='toolboxv2.mods.')

proxi attr

Source code in toolboxv2/utils/system/types.py

def reload_mod(self, mod_name, spec='app', is_file=True, loc="toolboxv2.mods."):
    """proxi attr"""

remove_mod(mod_name, spec='app', delete=True)

proxi attr

Source code in toolboxv2/utils/system/types.py

def remove_mod(self, mod_name, spec='app', delete=True):
    """proxi attr"""

rrun_flows(name, **kwargs)

proxi attr

Source code in toolboxv2/utils/system/types.py

def rrun_flows(self, name, **kwargs):
    """proxi attr"""

run_a_from_sync(function, *args)

run a async fuction

Source code in toolboxv2/utils/system/types.py

def run_a_from_sync(self, function, *args):
    """
    run a async fuction
    """

run_any(mod_function_name, backwords_compability_variabel_string_holder=None, get_results=False, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None, kwargs_=None, *args, **kwargs)

proxi attr

Source code in toolboxv2/utils/system/types.py

def run_any(self, mod_function_name: Enum or str or tuple, backwords_compability_variabel_string_holder=None,
            get_results=False, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None,
            kwargs_=None,
            *args, **kwargs):
    """proxi attr"""

run_bg_task(task)

run a async fuction

Source code in toolboxv2/utils/system/types.py

def run_bg_task(self, task):
    """
            run a async fuction
            """

run_bg_task_advanced(task, *args, **kwargs)

proxi attr

Source code in toolboxv2/utils/system/types.py

def run_bg_task_advanced(self, task, *args, **kwargs):
    """
    proxi attr
    """

run_flows(name, **kwargs) async

proxi attr

Source code in toolboxv2/utils/system/types.py

async def run_flows(self, name, **kwargs):
    """proxi attr"""

run_function(mod_function_name, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None, kwargs_=None, *args, **kwargs)

proxi attr

Source code in toolboxv2/utils/system/types.py

def run_function(self, mod_function_name: Enum or tuple,
                 tb_run_function_with_state=True,
                 tb_run_with_specification='app',
                 args_=None,
                 kwargs_=None,
                 *args,
                 **kwargs) -> Result:

    """proxi attr"""

run_http(mod_function_name, function_name=None, method='GET', args_=None, kwargs_=None, *args, **kwargs) async

run a function remote via http / https

Source code in toolboxv2/utils/system/types.py

async def run_http(self, mod_function_name: Enum or str or tuple, function_name=None, method="GET",
                   args_=None,
                   kwargs_=None,
                   *args, **kwargs):
    """run a function remote via http / https"""

save_autocompletion_dict()

proxi attr

Source code in toolboxv2/utils/system/types.py

def save_autocompletion_dict(self):
    """proxi attr"""

save_exit()

proxi attr

Source code in toolboxv2/utils/system/types.py

def save_exit(self):
    """proxi attr"""

save_initialized_module(tools_class, spec)

proxi attr

Source code in toolboxv2/utils/system/types.py

def save_initialized_module(self, tools_class, spec):
    """proxi attr"""

save_instance(instance, modular_id, spec='app', instance_type='file/application', tools_class=None)

proxi attr

Source code in toolboxv2/utils/system/types.py

def save_instance(self, instance, modular_id, spec='app', instance_type="file/application", tools_class=None):
    """proxi attr"""

save_load(modname, spec='app')

proxi attr

Source code in toolboxv2/utils/system/types.py

def save_load(self, modname, spec='app'):
    """proxi attr"""

save_registry_as_enums(directory, filename)

proxi attr

Source code in toolboxv2/utils/system/types.py

def save_registry_as_enums(self, directory: str, filename: str):
    """proxi attr"""

set_flows(r)

proxi attr

Source code in toolboxv2/utils/system/types.py

def set_flows(self, r):
    """proxi attr"""

set_logger(debug=False)

proxi attr

Source code in toolboxv2/utils/system/types.py

def set_logger(self, debug=False):
    """proxi attr"""

show_console(*args, **kwargs) async staticmethod

proxi attr

Source code in toolboxv2/utils/system/types.py

@staticmethod
async def show_console(*args, **kwargs):
    """proxi attr"""

sprint(text, *args, **kwargs) staticmethod

proxi attr

Source code in toolboxv2/utils/system/types.py

@staticmethod
def sprint(text, *args, **kwargs):
    """proxi attr"""

tb(name=None, mod_name='', helper='', version=None, test=True, restrict_in_virtual_mode=False, api=False, initial=False, exit_f=False, test_only=False, memory_cache=False, file_cache=False, row=False, request_as_kwarg=False, state=None, level=0, memory_cache_max_size=100, memory_cache_ttl=300, samples=None, interface=None, pre_compute=None, post_compute=None, api_methods=None, websocket_handler=None)

A decorator for registering and configuring functions within a module.

This decorator is used to wrap functions with additional functionality such as caching, API conversion, and lifecycle management (initialization and exit). It also handles the registration of the function in the module's function registry.

Parameters:

Name	Type	Description	Default
name	str	The name to register the function under. Defaults to the function's own name.	None
mod_name	str	The name of the module the function belongs to.	''
helper	str	A helper string providing additional information about the function.	''
version	str or None	The version of the function or module.	None
test	bool	Flag to indicate if the function is for testing purposes.	True
restrict_in_virtual_mode	bool	Flag to restrict the function in virtual mode.	False
api	bool	Flag to indicate if the function is part of an API.	False
initial	bool	Flag to indicate if the function should be executed at initialization.	False
exit_f	bool	Flag to indicate if the function should be executed at exit.	False
test_only	bool	Flag to indicate if the function should only be used for testing.	False
memory_cache	bool	Flag to enable memory caching for the function.	False
request_as_kwarg	bool	Flag to get request if the fuction is calld from api.	False
file_cache	bool	Flag to enable file caching for the function.	False
row	bool	rather to auto wrap the result in Result type default False means no row data aka result type	False
state	bool or None	Flag to indicate if the function maintains state.	None
level	int	The level of the function, used for prioritization or categorization.	0
memory_cache_max_size	int	Maximum size of the memory cache.	100
memory_cache_ttl	int	Time-to-live for the memory cache entries.	300
samples	list or dict or None	Samples or examples of function usage.	None
interface	str	The interface type for the function.	None
pre_compute	callable	A function to be called before the main function.	None
post_compute	callable	A function to be called after the main function.	None
api_methods	list[str]	default ["AUTO"] (GET if not params, POST if params) , GET, POST, PUT or DELETE.	None

Returns:

Name	Type	Description
function		The decorated function with additional processing and registration capabilities.

Source code in toolboxv2/utils/system/types.py

def tb(self, name=None,
       mod_name: str = "",
       helper: str = "",
       version: str or None = None,
       test: bool = True,
       restrict_in_virtual_mode: bool = False,
       api: bool = False,
       initial: bool = False,
       exit_f: bool = False,
       test_only: bool = False,
       memory_cache: bool = False,
       file_cache: bool = False,
       row=False,
       request_as_kwarg: bool = False,
       state: bool or None = None,
       level: int = 0,
       memory_cache_max_size: int = 100,
       memory_cache_ttl: int = 300,
       samples: list or dict or None = None,
       interface: ToolBoxInterfaces or None or str = None,
       pre_compute=None,
       post_compute=None,
       api_methods=None,
       websocket_handler: str | None = None,
       ):
    """
A decorator for registering and configuring functions within a module.

This decorator is used to wrap functions with additional functionality such as caching, API conversion, and lifecycle management (initialization and exit). It also handles the registration of the function in the module's function registry.

Args:
    name (str, optional): The name to register the function under. Defaults to the function's own name.
    mod_name (str, optional): The name of the module the function belongs to.
    helper (str, optional): A helper string providing additional information about the function.
    version (str or None, optional): The version of the function or module.
    test (bool, optional): Flag to indicate if the function is for testing purposes.
    restrict_in_virtual_mode (bool, optional): Flag to restrict the function in virtual mode.
    api (bool, optional): Flag to indicate if the function is part of an API.
    initial (bool, optional): Flag to indicate if the function should be executed at initialization.
    exit_f (bool, optional): Flag to indicate if the function should be executed at exit.
    test_only (bool, optional): Flag to indicate if the function should only be used for testing.
    memory_cache (bool, optional): Flag to enable memory caching for the function.
    request_as_kwarg (bool, optional): Flag to get request if the fuction is calld from api.
    file_cache (bool, optional): Flag to enable file caching for the function.
    row (bool, optional): rather to auto wrap the result in Result type default False means no row data aka result type
    state (bool or None, optional): Flag to indicate if the function maintains state.
    level (int, optional): The level of the function, used for prioritization or categorization.
    memory_cache_max_size (int, optional): Maximum size of the memory cache.
    memory_cache_ttl (int, optional): Time-to-live for the memory cache entries.
    samples (list or dict or None, optional): Samples or examples of function usage.
    interface (str, optional): The interface type for the function.
    pre_compute (callable, optional): A function to be called before the main function.
    post_compute (callable, optional): A function to be called after the main function.
    api_methods (list[str], optional): default ["AUTO"] (GET if not params, POST if params) , GET, POST, PUT or DELETE.

Returns:
    function: The decorated function with additional processing and registration capabilities.
"""
    if interface is None:
        interface = "tb"
    if test_only and 'test' not in self.id:
        return lambda *args, **kwargs: args
    return self._create_decorator(interface,
                                  name,
                                  mod_name,
                                  level=level,
                                  restrict_in_virtual_mode=restrict_in_virtual_mode,
                                  helper=helper,
                                  api=api,
                                  version=version,
                                  initial=initial,
                                  exit_f=exit_f,
                                  test=test,
                                  samples=samples,
                                  state=state,
                                  pre_compute=pre_compute,
                                  post_compute=post_compute,
                                  memory_cache=memory_cache,
                                  file_cache=file_cache,
                                  row=row,
                                  request_as_kwarg=request_as_kwarg,
                                  memory_cache_max_size=memory_cache_max_size,
                                  memory_cache_ttl=memory_cache_ttl)

wait_for_bg_tasks(timeout=None)

proxi attr

Source code in toolboxv2/utils/system/types.py

def wait_for_bg_tasks(self, timeout=None):
    """
    proxi attr
    """

watch_mod(mod_name, spec='app', loc='toolboxv2.mods.', use_thread=True, path_name=None)

proxi attr

Source code in toolboxv2/utils/system/types.py

def watch_mod(self, mod_name, spec='app', loc="toolboxv2.mods.", use_thread=True, path_name=None):
    """proxi attr"""

web_context()

returns the build index ( toolbox web component )

Source code in toolboxv2/utils/system/types.py

def web_context(self) -> str:
    """returns the build index ( toolbox web component )"""

MainTool

Source code in toolboxv2/utils/system/main_tool.py

class MainTool:
    toolID: str = ""
    # app = None
    interface = None
    spec = "app"
    name = ""
    color = "Bold"
    stuf = False

    def __init__(self, *args, **kwargs):
        """
        Standard constructor used for arguments pass
        Do not override. Use __ainit__ instead
        """
        self.__storedargs = args, kwargs
        self.tools = kwargs.get("tool", {})
        self.logger = kwargs.get("logs", get_logger())
        self.color = kwargs.get("color", "WHITE")
        self.todo = kwargs.get("load", kwargs.get("on_start", lambda: None))
        if "on_exit" in kwargs and isinstance(kwargs.get("on_exit"), Callable):
            self.on_exit =self.app.tb(
                mod_name=self.name,
                name=kwargs.get("on_exit").__name__,
                version=self.version if hasattr(self, 'version') else "0.0.0",
            )(kwargs.get("on_exit"))
        self.async_initialized = False
        if self.todo:
            try:
                if inspect.iscoroutinefunction(self.todo):
                    pass
                else:
                    self.todo()
                get_logger().info(f"{self.name} on load suspended")
            except Exception as e:
                get_logger().error(f"Error loading mod {self.name} {e}")
                if self.app.debug:
                    import traceback
                    traceback.print_exc()
        else:
            get_logger().info(f"{self.name} no load require")

    async def __ainit__(self, *args, **kwargs):
        self.version = kwargs.get("v", kwargs.get("version", "0.0.0"))
        self.tools = kwargs.get("tool", {})
        self.name = kwargs["name"]
        self.logger = kwargs.get("logs", get_logger())
        self.color = kwargs.get("color", "WHITE")
        self.todo = kwargs.get("load", kwargs.get("on_start"))
        if not hasattr(self, 'config'):
            self.config = {}
        self.user = None
        self.description = "A toolbox mod" if kwargs.get("description") is None else kwargs.get("description")
        if MainTool.interface is None:
            MainTool.interface = self.app.interface_type
        # Result.default(self.app.interface)

        if self.todo:
            try:
                if inspect.iscoroutinefunction(self.todo):
                    await self.todo()
                else:
                    pass
                await asyncio.sleep(0.1)
                get_logger().info(f"{self.name} on load suspended")
            except Exception as e:
                get_logger().error(f"Error loading mod {self.name} {e}")
                if self.app.debug:
                    import traceback
                    traceback.print_exc()
        else:
            get_logger().info(f"{self.name} no load require")
        self.app.print(f"TOOL : {self.spec}.{self.name} online")



    @property
    def app(self):
        return get_app(
            from_=f"{self.spec}.{self.name}|{self.toolID if self.toolID else '*' + MainTool.toolID} {self.interface if self.interface else MainTool.interface}")

    @app.setter
    def app(self, v):
        raise PermissionError(f"You cannot set the App Instance! {v=}")

    @staticmethod
    def return_result(error: ToolBoxError = ToolBoxError.none,
                      exec_code: int = 0,
                      help_text: str = "",
                      data_info=None,
                      data=None,
                      data_to=None):

        if data_to is None:
            data_to = MainTool.interface if MainTool.interface is not None else ToolBoxInterfaces.cli

        if data is None:
            data = {}

        if data_info is None:
            data_info = {}

        return Result(
            error,
            ToolBoxResult(data_info=data_info, data=data, data_to=data_to),
            ToolBoxInfo(exec_code=exec_code, help_text=help_text)
        )

    def print(self, message, end="\n", **kwargs):
        if self.stuf:
            return

        self.app.print(Style.style_dic[self.color] + self.name + Style.style_dic["END"] + ":", message, end=end,
                       **kwargs)

    def add_str_to_config(self, command):
        if len(command) != 2:
            self.logger.error('Invalid command must be key value')
            return False
        self.config[command[0]] = command[1]

    def webInstall(self, user_instance, construct_render) -> str:
        """"Returns a web installer for the given user instance and construct render template"""

    def get_version(self) -> str:
        """"Returns the version"""
        return self.version

    async def get_user(self, username: str) -> Result:
        return await self.app.a_run_any(CLOUDM_AUTHMANAGER.GET_USER_BY_NAME, username=username, get_results=True)

    async def __initobj(self):
        """Crutch used for __await__ after spawning"""
        assert not self.async_initialized
        self.async_initialized = True
        # pass the parameters to __ainit__ that passed to __init__
        await self.__ainit__(*self.__storedargs[0], **self.__storedargs[1])
        return self

    def __await__(self):
        return self.__initobj().__await__()

__init__(*args, **kwargs)

Standard constructor used for arguments pass Do not override. Use ainit instead

Source code in toolboxv2/utils/system/main_tool.py

def __init__(self, *args, **kwargs):
    """
    Standard constructor used for arguments pass
    Do not override. Use __ainit__ instead
    """
    self.__storedargs = args, kwargs
    self.tools = kwargs.get("tool", {})
    self.logger = kwargs.get("logs", get_logger())
    self.color = kwargs.get("color", "WHITE")
    self.todo = kwargs.get("load", kwargs.get("on_start", lambda: None))
    if "on_exit" in kwargs and isinstance(kwargs.get("on_exit"), Callable):
        self.on_exit =self.app.tb(
            mod_name=self.name,
            name=kwargs.get("on_exit").__name__,
            version=self.version if hasattr(self, 'version') else "0.0.0",
        )(kwargs.get("on_exit"))
    self.async_initialized = False
    if self.todo:
        try:
            if inspect.iscoroutinefunction(self.todo):
                pass
            else:
                self.todo()
            get_logger().info(f"{self.name} on load suspended")
        except Exception as e:
            get_logger().error(f"Error loading mod {self.name} {e}")
            if self.app.debug:
                import traceback
                traceback.print_exc()
    else:
        get_logger().info(f"{self.name} no load require")

__initobj() async

Crutch used for await after spawning

Source code in toolboxv2/utils/system/main_tool.py

async def __initobj(self):
    """Crutch used for __await__ after spawning"""
    assert not self.async_initialized
    self.async_initialized = True
    # pass the parameters to __ainit__ that passed to __init__
    await self.__ainit__(*self.__storedargs[0], **self.__storedargs[1])
    return self

get_version()

"Returns the version

Source code in toolboxv2/utils/system/main_tool.py

def get_version(self) -> str:
    """"Returns the version"""
    return self.version

webInstall(user_instance, construct_render)

"Returns a web installer for the given user instance and construct render template

Source code in toolboxv2/utils/system/main_tool.py

def webInstall(self, user_instance, construct_render) -> str:
    """"Returns a web installer for the given user instance and construct render template"""

MainToolType

Source code in toolboxv2/utils/system/types.py

class MainToolType:
    toolID: str
    app: A
    interface: ToolBoxInterfaces
    spec: str

    version: str
    tools: dict  # legacy
    name: str
    logger: logging
    color: str
    todo: Callable
    _on_exit: Callable
    stuf: bool
    config: dict
    user: U | None
    description: str

    @staticmethod
    def return_result(error: ToolBoxError = ToolBoxError.none,
                      exec_code: int = 0,
                      help_text: str = "",
                      data_info=None,
                      data=None,
                      data_to=None) -> Result:
        """proxi attr"""

    def load(self):
        """proxi attr"""

    def print(self, message, end="\n", **kwargs):
        """proxi attr"""

    def add_str_to_config(self, command):
        if len(command) != 2:
            self.logger.error('Invalid command must be key value')
            return False
        self.config[command[0]] = command[1]

    def webInstall(self, user_instance, construct_render) -> str:
        """"Returns a web installer for the given user instance and construct render template"""

    async def get_user(self, username: str) -> Result:
        return self.app.a_run_any(CLOUDM_AUTHMANAGER.GET_USER_BY_NAME, username=username, get_results=True)

load()

proxi attr

Source code in toolboxv2/utils/system/types.py

def load(self):
    """proxi attr"""

print(message, end='\n', **kwargs)

proxi attr

Source code in toolboxv2/utils/system/types.py

def print(self, message, end="\n", **kwargs):
    """proxi attr"""

return_result(error=ToolBoxError.none, exec_code=0, help_text='', data_info=None, data=None, data_to=None) staticmethod

proxi attr

Source code in toolboxv2/utils/system/types.py

@staticmethod
def return_result(error: ToolBoxError = ToolBoxError.none,
                  exec_code: int = 0,
                  help_text: str = "",
                  data_info=None,
                  data=None,
                  data_to=None) -> Result:
    """proxi attr"""

webInstall(user_instance, construct_render)

"Returns a web installer for the given user instance and construct render template

Source code in toolboxv2/utils/system/types.py

def webInstall(self, user_instance, construct_render) -> str:
    """"Returns a web installer for the given user instance and construct render template"""

Result

Source code in toolboxv2/utils/system/types.py

class Result(Generic[T]):
    _task = None
    _generic_type: Optional[Type] = None

    def __init__(self,
                 error: ToolBoxError,
                 result: ToolBoxResult,
                 info: ToolBoxInfo,
                 origin: Any | None = None,
                 generic_type: Optional[Type] = None
                 ):
        self.error: ToolBoxError = error
        self.result: ToolBoxResult = result
        self.info: ToolBoxInfo = info
        self.origin = origin
        self._generic_type = generic_type

    def __class_getitem__(cls, item):
        """Enable Result[Type] syntax"""

        class TypedResult(cls):
            _generic_type = item

            def __init__(self, *args, **kwargs):
                super().__init__(*args, **kwargs)
                self._generic_type = item

        return TypedResult

    def typed_get(self, key=None, default=None) -> T:
        """Get data with type validation"""
        data = self.get(key, default)

        if self._generic_type and data is not None:
            # Validate type matches generic parameter
            if not self._validate_type(data, self._generic_type):
                from toolboxv2 import get_logger
                get_logger().warning(f"Type mismatch: expected {self._generic_type}, got {type(data)}")

        return data

    async def typed_aget(self, key=None, default=None) -> T:
        """Async get data with type validation"""
        data = await self.aget(key, default)

        if self._generic_type and data is not None:
            if not self._validate_type(data, self._generic_type):
                from toolboxv2 import get_logger
                get_logger().warning(f"Type mismatch: expected {self._generic_type}, got {type(data)}")

        return data

    def _validate_type(self, data, expected_type) -> bool:
        """Validate data matches expected type"""
        try:
            # Handle List[Type] syntax
            origin = get_origin(expected_type)
            if origin is list or origin is List:
                if not isinstance(data, list):
                    return False

                # Check list element types if specified
                args = get_args(expected_type)
                if args and data:
                    element_type = args[0]
                    return all(isinstance(item, element_type) for item in data)
                return True

            # Handle other generic types
            elif origin is not None:
                return isinstance(data, origin)

            # Handle regular types
            else:
                return isinstance(data, expected_type)

        except Exception:
            return True  # Skip validation on error

    @classmethod
    def typed_ok(cls, data: T, data_info="", info="OK", interface=ToolBoxInterfaces.native) -> 'Result[T]':
        """Create OK result with type information"""
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=0, help_text=info)
        result = ToolBoxResult(data_to=interface, data=data, data_info=data_info, data_type=type(data).__name__)

        instance = cls(error=error, info=info_obj, result=result)
        if hasattr(cls, '_generic_type'):
            instance._generic_type = cls._generic_type

        return instance

    @classmethod
    def typed_json(cls, data: T, info="OK", interface=ToolBoxInterfaces.remote, exec_code=0,
                   status_code=None) -> 'Result[T]':
        """Create JSON result with type information"""
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=status_code or exec_code, help_text=info)

        result = ToolBoxResult(
            data_to=interface,
            data=data,
            data_info="JSON response",
            data_type="json"
        )

        instance = cls(error=error, info=info_obj, result=result)
        if hasattr(cls, '_generic_type'):
            instance._generic_type = cls._generic_type

        return instance

    def cast_to(self, target_type: Type[T]) -> 'Result[T]':
        """Cast result to different type"""
        new_result = Result(
            error=self.error,
            result=self.result,
            info=self.info,
            origin=self.origin,
            generic_type=target_type
        )
        new_result._generic_type = target_type
        return new_result

    def get_type_info(self) -> Optional[Type]:
        """Get the generic type information"""
        return self._generic_type

    def is_typed(self) -> bool:
        """Check if result has type information"""
        return self._generic_type is not None

    def as_result(self):
        return self

    def as_dict(self):
        return {
            "error":self.error.value if isinstance(self.error, Enum) else self.error,
        "result" : {
            "data_to":self.result.data_to.value if isinstance(self.result.data_to, Enum) else self.result.data_to,
            "data_info":self.result.data_info,
            "data":self.result.data,
            "data_type":self.result.data_type
        } if self.result else None,
        "info" : {
            "exec_code" : self.info.exec_code,  # exec_code umwandel in http resposn codes
        "help_text" : self.info.help_text
        } if self.info else None,
        "origin" : self.origin
        }

    def set_origin(self, origin):
        if self.origin is not None:
            raise ValueError("You cannot Change the origin of a Result!")
        self.origin = origin
        return self

    def set_dir_origin(self, name, extras="assets/"):
        if self.origin is not None:
            raise ValueError("You cannot Change the origin of a Result!")
        self.origin = f"mods/{name}/{extras}"
        return self

    def is_error(self):
        if _test_is_result(self.result.data):
            return self.result.data.is_error()
        if self.error == ToolBoxError.none:
            return False
        if self.info.exec_code == 0:
            return False
        return self.info.exec_code != 200

    def is_ok(self):
        return not self.is_error()

    def is_data(self):
        return self.result.data is not None

    def to_api_result(self):
        # print(f" error={self.error}, result= {self.result}, info= {self.info}, origin= {self.origin}")
        return ApiResult(
            error=self.error.value if isinstance(self.error, Enum) else self.error,
            result=ToolBoxResultBM(
                data_to=self.result.data_to.value if isinstance(self.result.data_to, Enum) else self.result.data_to,
                data_info=self.result.data_info,
                data=self.result.data,
                data_type=self.result.data_type
            ) if self.result else None,
            info=ToolBoxInfoBM(
                exec_code=self.info.exec_code,  # exec_code umwandel in http resposn codes
                help_text=self.info.help_text
            ) if self.info else None,
            origin=self.origin
        )

    def task(self, task):
        self._task = task
        return self

    @staticmethod
    def result_from_dict(error: str, result: dict, info: dict, origin: list or None or str):
        # print(f" error={self.error}, result= {self.result}, info= {self.info}, origin= {self.origin}")
        return ApiResult(
            error=error if isinstance(error, Enum) else error,
            result=ToolBoxResultBM(
                data_to=result.get('data_to') if isinstance(result.get('data_to'), Enum) else result.get('data_to'),
                data_info=result.get('data_info', '404'),
                data=result.get('data'),
                data_type=result.get('data_type', '404'),
            ) if result else ToolBoxResultBM(
                data_to=ToolBoxInterfaces.cli.value,
                data_info='',
                data='404',
                data_type='404',
            ),
            info=ToolBoxInfoBM(
                exec_code=info.get('exec_code', 404),
                help_text=info.get('help_text', '404')
            ) if info else ToolBoxInfoBM(
                exec_code=404,
                help_text='404'
            ),
            origin=origin
        ).as_result()

    @classmethod
    def stream(cls,
               stream_generator: Any,  # Renamed from source for clarity
               content_type: str = "text/event-stream",  # Default to SSE
               headers: dict | None = None,
               info: str = "OK",
               interface: ToolBoxInterfaces = ToolBoxInterfaces.remote,
               cleanup_func: Callable[[], None] | Callable[[], T] | Callable[[], AsyncGenerator[T, None]] | None = None):
        """
        Create a streaming response Result. Handles SSE and other stream types.

        Args:
            stream_generator: Any stream source (async generator, sync generator, iterable, or single item).
            content_type: Content-Type header (default: text/event-stream for SSE).
            headers: Additional HTTP headers for the response.
            info: Help text for the result.
            interface: Interface to send data to.
            cleanup_func: Optional function for cleanup.

        Returns:
            A Result object configured for streaming.
        """
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=0, help_text=info)

        final_generator: AsyncGenerator[str, None]

        if content_type == "text/event-stream":
            # For SSE, always use SSEGenerator.create_sse_stream to wrap the source.
            # SSEGenerator.create_sse_stream handles various types of stream_generator internally.
            final_generator = SSEGenerator.create_sse_stream(source=stream_generator, cleanup_func=cleanup_func)

            # Standard SSE headers for the HTTP response itself
            # These will be stored in the Result object. Rust side decides how to use them.
            standard_sse_headers = {
                "Cache-Control": "no-cache",  # SSE specific
                "Connection": "keep-alive",  # SSE specific
                "X-Accel-Buffering": "no",  # Useful for proxies with SSE
                # Content-Type is implicitly text/event-stream, will be in streaming_data below
            }
            all_response_headers = standard_sse_headers.copy()
            if headers:
                all_response_headers.update(headers)
        else:
            # For non-SSE streams.
            # If stream_generator is sync, wrap it to be async.
            # If already async or single item, it will be handled.
            # Rust's stream_generator in ToolboxClient seems to handle both sync/async Python generators.
            # For consistency with how SSEGenerator does it, we can wrap sync ones.
            if inspect.isgenerator(stream_generator) or \
                (not isinstance(stream_generator, str) and hasattr(stream_generator, '__iter__')):
                final_generator = SSEGenerator.wrap_sync_generator(stream_generator)  # Simple async wrapper
            elif inspect.isasyncgen(stream_generator):
                final_generator = stream_generator
            else:  # Single item or string
                async def _single_item_gen():
                    yield stream_generator

                final_generator = _single_item_gen()
            all_response_headers = headers if headers else {}

        # Prepare streaming data to be stored in the Result object
        streaming_data = {
            "type": "stream",  # Indicator for Rust side
            "generator": final_generator,
            "content_type": content_type,  # Let Rust know the intended content type
            "headers": all_response_headers  # Intended HTTP headers for the overall response
        }

        result_payload = ToolBoxResult(
            data_to=interface,
            data=streaming_data,
            data_info="Streaming response" if content_type != "text/event-stream" else "SSE Event Stream",
            data_type="stream"  # Generic type for Rust to identify it needs to stream from 'generator'
        )

        return cls(error=error, info=info_obj, result=result_payload)

    @classmethod
    def sse(cls,
            stream_generator: Any,
            info: str = "OK",
            interface: ToolBoxInterfaces = ToolBoxInterfaces.remote,
            cleanup_func: Callable[[], None] | Callable[[], T] | Callable[[], AsyncGenerator[T, None]] | None = None,
            # http_headers: Optional[dict] = None # If we want to allow overriding default SSE HTTP headers
            ):
        """
        Create an Server-Sent Events (SSE) streaming response Result.

        Args:
            stream_generator: A source yielding individual data items. This can be an
                              async generator, sync generator, iterable, or a single item.
                              Each item will be formatted as an SSE event.
            info: Optional help text for the Result.
            interface: Optional ToolBoxInterface to target.
            cleanup_func: Optional cleanup function to run when the stream ends or is cancelled.
            #http_headers: Optional dictionary of custom HTTP headers for the SSE response.

        Returns:
            A Result object configured for SSE streaming.
        """
        # Result.stream will handle calling SSEGenerator.create_sse_stream
        # and setting appropriate default headers for SSE when content_type is "text/event-stream".
        return cls.stream(
            stream_generator=stream_generator,
            content_type="text/event-stream",
            # headers=http_headers, # Pass if we add http_headers param
            info=info,
            interface=interface,
            cleanup_func=cleanup_func
        )

    @classmethod
    def default(cls, interface=ToolBoxInterfaces.native):
        error = ToolBoxError.none
        info = ToolBoxInfo(exec_code=-1, help_text="")
        result = ToolBoxResult(data_to=interface)
        return cls(error=error, info=info, result=result)

    @classmethod
    def json(cls, data, info="OK", interface=ToolBoxInterfaces.remote, exec_code=0, status_code=None):
        """Create a JSON response Result."""
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=status_code or exec_code, help_text=info)

        result = ToolBoxResult(
            data_to=interface,
            data=data,
            data_info="JSON response",
            data_type="json"
        )

        return cls(error=error, info=info_obj, result=result)

    @classmethod
    def text(cls, text_data, content_type="text/plain",exec_code=None,status=200, info="OK", interface=ToolBoxInterfaces.remote, headers=None):
        """Create a text response Result with specific content type."""
        if headers is not None:
            return cls.html(text_data, status= exec_code or status, info=info, headers=headers)
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=exec_code or status, help_text=info)

        result = ToolBoxResult(
            data_to=interface,
            data=text_data,
            data_info="Text response",
            data_type=content_type
        )

        return cls(error=error, info=info_obj, result=result)

    @classmethod
    def binary(cls, data, content_type="application/octet-stream", download_name=None, info="OK",
               interface=ToolBoxInterfaces.remote):
        """Create a binary data response Result."""
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=0, help_text=info)

        # Create a dictionary with binary data and metadata
        binary_data = {
            "data": data,
            "content_type": content_type,
            "filename": download_name
        }

        result = ToolBoxResult(
            data_to=interface,
            data=binary_data,
            data_info=f"Binary response: {download_name}" if download_name else "Binary response",
            data_type="binary"
        )

        return cls(error=error, info=info_obj, result=result)

    @classmethod
    def file(cls, data, filename, content_type=None, info="OK", interface=ToolBoxInterfaces.remote):
        """Create a file download response Result.

        Args:
            data: File data as bytes or base64 string
            filename: Name of the file for download
            content_type: MIME type of the file (auto-detected if None)
            info: Response info text
            interface: Target interface

        Returns:
            Result object configured for file download
        """
        import base64
        import mimetypes

        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=200, help_text=info)

        # Auto-detect content type if not provided
        if content_type is None:
            content_type, _ = mimetypes.guess_type(filename)
            if content_type is None:
                content_type = "application/octet-stream"

        # Ensure data is base64 encoded string (as expected by Rust server)
        if isinstance(data, bytes):
            base64_data = base64.b64encode(data).decode('utf-8')
        elif isinstance(data, str):
            # Assume it's already base64 encoded
            base64_data = data
        else:
            raise ValueError("File data must be bytes or base64 string")

        result = ToolBoxResult(
            data_to=interface,
            data=base64_data,  # Rust expects base64 string for "file" type
            data_info=f"File download: {filename}",
            data_type="file"
        )

        return cls(error=error, info=info_obj, result=result)

    @classmethod
    def redirect(cls, url, status_code=302, info="Redirect", interface=ToolBoxInterfaces.remote):
        """Create a redirect response."""
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=status_code, help_text=info)

        result = ToolBoxResult(
            data_to=interface,
            data=url,
            data_info="Redirect response",
            data_type="redirect"
        )

        return cls(error=error, info=info_obj, result=result)

    @classmethod
    def ok(cls, data=None, data_info="", info="OK", interface=ToolBoxInterfaces.native):
        error = ToolBoxError.none
        info = ToolBoxInfo(exec_code=0, help_text=info)
        result = ToolBoxResult(data_to=interface, data=data, data_info=data_info, data_type=type(data).__name__)
        return cls(error=error, info=info, result=result)

    @classmethod
    def html(cls, data=None, data_info="", info="OK", interface=ToolBoxInterfaces.remote, data_type="html",status=200, headers=None, row=False):
        error = ToolBoxError.none
        info = ToolBoxInfo(exec_code=status, help_text=info)
        from ...utils.system.getting_and_closing_app import get_app

        if not row and not '"<div class="main-content""' in data:
            data = f'<div class="main-content frosted-glass">{data}<div>'
        if not row and not get_app().web_context() in data:
            data = get_app().web_context() + data

        if isinstance(headers, dict):
            result = ToolBoxResult(data_to=interface, data={'html':data,'headers':headers}, data_info=data_info,
                                   data_type="special_html")
        else:
            result = ToolBoxResult(data_to=interface, data=data, data_info=data_info,
                                   data_type=data_type if data_type is not None else type(data).__name__)
        return cls(error=error, info=info, result=result)

    @classmethod
    def future(cls, data=None, data_info="", info="OK", interface=ToolBoxInterfaces.future):
        error = ToolBoxError.none
        info = ToolBoxInfo(exec_code=0, help_text=info)
        result = ToolBoxResult(data_to=interface, data=data, data_info=data_info, data_type="future")
        return cls(error=error, info=info, result=result)

    @classmethod
    def custom_error(cls, data=None, data_info="", info="", exec_code=-1, interface=ToolBoxInterfaces.native):
        error = ToolBoxError.custom_error
        info = ToolBoxInfo(exec_code=exec_code, help_text=info)
        result = ToolBoxResult(data_to=interface, data=data, data_info=data_info, data_type=type(data).__name__)
        return cls(error=error, info=info, result=result)

    @classmethod
    def error(cls, data=None, data_info="", info="", exec_code=450, interface=ToolBoxInterfaces.remote):
        error = ToolBoxError.custom_error
        info = ToolBoxInfo(exec_code=exec_code, help_text=info)
        result = ToolBoxResult(data_to=interface, data=data, data_info=data_info, data_type=type(data).__name__)
        return cls(error=error, info=info, result=result)

    @classmethod
    def default_user_error(cls, info="", exec_code=-3, interface=ToolBoxInterfaces.native, data=None):
        error = ToolBoxError.input_error
        info = ToolBoxInfo(exec_code, info)
        result = ToolBoxResult(data_to=interface, data=data, data_type=type(data).__name__)
        return cls(error=error, info=info, result=result)

    @classmethod
    def default_internal_error(cls, info="", exec_code=-2, interface=ToolBoxInterfaces.native, data=None):
        error = ToolBoxError.internal_error
        info = ToolBoxInfo(exec_code, info)
        result = ToolBoxResult(data_to=interface, data=data, data_type=type(data).__name__)
        return cls(error=error, info=info, result=result)

    def print(self, show=True, show_data=True, prifix="", full_data=False):
        data = '\n' + f"{((prifix + f'Data_{self.result.data_type}: ' + str(self.result.data) if self.result.data is not None else 'NO Data') if not isinstance(self.result.data, Result) else self.result.data.print(show=False, show_data=show_data, prifix=prifix + '-')) if show_data else 'Data: private'}"
        origin = '\n' + f"{prifix + 'Origin: ' + str(self.origin) if self.origin is not None else 'NO Origin'}"
        text = (f"Function Exec code: {self.info.exec_code}"
                f"\n{prifix}Info's:"
                f" {self.info.help_text} {'<|> ' + str(self.result.data_info) if self.result.data_info is not None else ''}"
                f"{origin}{((data[:100]+'...') if not full_data else (data)) if not data.endswith('NO Data') else ''}\n")
        if not show:
            return text
        print("\n======== Result ========\n" + text + "------- EndOfD -------")
        return self

    def log(self, show_data=True, prifix=""):
        from toolboxv2 import get_logger
        get_logger().debug(self.print(show=False, show_data=show_data, prifix=prifix).replace("\n", " - "))
        return self

    def __str__(self):
        return self.print(show=False, show_data=True)

    def get(self, key=None, default=None):
        data = self.result.data
        if isinstance(data, Result):
            return data.get(key=key, default=default)
        if key is not None and isinstance(data, dict):
            return data.get(key, default)
        return data if data is not None else default

    async def aget(self, key=None, default=None):
        if asyncio.isfuture(self.result.data) or asyncio.iscoroutine(self.result.data) or (
            isinstance(self.result.data_to, Enum) and self.result.data_to.name == ToolBoxInterfaces.future.name):
            data = await self.result.data
        else:
            data = self.get(key=None, default=None)
        if isinstance(data, Result):
            return data.get(key=key, default=default)
        if key is not None and isinstance(data, dict):
            return data.get(key, default)
        return data if data is not None else default

    def lazy_return(self, _=0, data=None, **kwargs):
        flags = ['raise', 'logg', 'user', 'intern']
        flag = flags[_] if isinstance(_, int) else _
        if self.info.exec_code == 0:
            return self if data is None else data if _test_is_result(data) else self.ok(data=data, **kwargs)
        if flag == 'raise':
            raise ValueError(self.print(show=False))
        if flag == 'logg':
            from .. import get_logger
            get_logger().error(self.print(show=False))

        if flag == 'user':
            return self if data is None else data if _test_is_result(data) else self.default_user_error(data=data,
                                                                                                        **kwargs)
        if flag == 'intern':
            return self if data is None else data if _test_is_result(data) else self.default_internal_error(data=data,
                                                                                                            **kwargs)

        return self if data is None else data if _test_is_result(data) else self.custom_error(data=data, **kwargs)

    @property
    def bg_task(self):
        return self._task

__class_getitem__(item)

Enable Result[Type] syntax

Source code in toolboxv2/utils/system/types.py

def __class_getitem__(cls, item):
    """Enable Result[Type] syntax"""

    class TypedResult(cls):
        _generic_type = item

        def __init__(self, *args, **kwargs):
            super().__init__(*args, **kwargs)
            self._generic_type = item

    return TypedResult

binary(data, content_type='application/octet-stream', download_name=None, info='OK', interface=ToolBoxInterfaces.remote) classmethod

Create a binary data response Result.

Source code in toolboxv2/utils/system/types.py

@classmethod
def binary(cls, data, content_type="application/octet-stream", download_name=None, info="OK",
           interface=ToolBoxInterfaces.remote):
    """Create a binary data response Result."""
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=0, help_text=info)

    # Create a dictionary with binary data and metadata
    binary_data = {
        "data": data,
        "content_type": content_type,
        "filename": download_name
    }

    result = ToolBoxResult(
        data_to=interface,
        data=binary_data,
        data_info=f"Binary response: {download_name}" if download_name else "Binary response",
        data_type="binary"
    )

    return cls(error=error, info=info_obj, result=result)

cast_to(target_type)

Cast result to different type

Source code in toolboxv2/utils/system/types.py

def cast_to(self, target_type: Type[T]) -> 'Result[T]':
    """Cast result to different type"""
    new_result = Result(
        error=self.error,
        result=self.result,
        info=self.info,
        origin=self.origin,
        generic_type=target_type
    )
    new_result._generic_type = target_type
    return new_result

file(data, filename, content_type=None, info='OK', interface=ToolBoxInterfaces.remote) classmethod

Create a file download response Result.

Parameters:

Name	Type	Description	Default
data		File data as bytes or base64 string	required
filename		Name of the file for download	required
content_type		MIME type of the file (auto-detected if None)	None
info		Response info text	'OK'
interface		Target interface	remote

Returns:

Type	Description
	Result object configured for file download

Source code in toolboxv2/utils/system/types.py

@classmethod
def file(cls, data, filename, content_type=None, info="OK", interface=ToolBoxInterfaces.remote):
    """Create a file download response Result.

    Args:
        data: File data as bytes or base64 string
        filename: Name of the file for download
        content_type: MIME type of the file (auto-detected if None)
        info: Response info text
        interface: Target interface

    Returns:
        Result object configured for file download
    """
    import base64
    import mimetypes

    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=200, help_text=info)

    # Auto-detect content type if not provided
    if content_type is None:
        content_type, _ = mimetypes.guess_type(filename)
        if content_type is None:
            content_type = "application/octet-stream"

    # Ensure data is base64 encoded string (as expected by Rust server)
    if isinstance(data, bytes):
        base64_data = base64.b64encode(data).decode('utf-8')
    elif isinstance(data, str):
        # Assume it's already base64 encoded
        base64_data = data
    else:
        raise ValueError("File data must be bytes or base64 string")

    result = ToolBoxResult(
        data_to=interface,
        data=base64_data,  # Rust expects base64 string for "file" type
        data_info=f"File download: {filename}",
        data_type="file"
    )

    return cls(error=error, info=info_obj, result=result)

get_type_info()

Get the generic type information

Source code in toolboxv2/utils/system/types.py

def get_type_info(self) -> Optional[Type]:
    """Get the generic type information"""
    return self._generic_type

is_typed()

Check if result has type information

Source code in toolboxv2/utils/system/types.py

def is_typed(self) -> bool:
    """Check if result has type information"""
    return self._generic_type is not None

json(data, info='OK', interface=ToolBoxInterfaces.remote, exec_code=0, status_code=None) classmethod

Create a JSON response Result.

Source code in toolboxv2/utils/system/types.py

@classmethod
def json(cls, data, info="OK", interface=ToolBoxInterfaces.remote, exec_code=0, status_code=None):
    """Create a JSON response Result."""
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=status_code or exec_code, help_text=info)

    result = ToolBoxResult(
        data_to=interface,
        data=data,
        data_info="JSON response",
        data_type="json"
    )

    return cls(error=error, info=info_obj, result=result)

redirect(url, status_code=302, info='Redirect', interface=ToolBoxInterfaces.remote) classmethod

Create a redirect response.

Source code in toolboxv2/utils/system/types.py

@classmethod
def redirect(cls, url, status_code=302, info="Redirect", interface=ToolBoxInterfaces.remote):
    """Create a redirect response."""
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=status_code, help_text=info)

    result = ToolBoxResult(
        data_to=interface,
        data=url,
        data_info="Redirect response",
        data_type="redirect"
    )

    return cls(error=error, info=info_obj, result=result)

sse(stream_generator, info='OK', interface=ToolBoxInterfaces.remote, cleanup_func=None) classmethod

Create an Server-Sent Events (SSE) streaming response Result.

Parameters:

Name	Type	Description	Default
stream_generator	Any	A source yielding individual data items. This can be an async generator, sync generator, iterable, or a single item. Each item will be formatted as an SSE event.	required
info	str	Optional help text for the Result.	'OK'
interface	ToolBoxInterfaces	Optional ToolBoxInterface to target.	remote
cleanup_func	Callable[[], None] | Callable[[], T] | Callable[[], AsyncGenerator[T, None]] | None	Optional cleanup function to run when the stream ends or is cancelled.	None
#http_headers		Optional dictionary of custom HTTP headers for the SSE response.	required

Returns:

Type	Description
	A Result object configured for SSE streaming.

Source code in toolboxv2/utils/system/types.py

@classmethod
def sse(cls,
        stream_generator: Any,
        info: str = "OK",
        interface: ToolBoxInterfaces = ToolBoxInterfaces.remote,
        cleanup_func: Callable[[], None] | Callable[[], T] | Callable[[], AsyncGenerator[T, None]] | None = None,
        # http_headers: Optional[dict] = None # If we want to allow overriding default SSE HTTP headers
        ):
    """
    Create an Server-Sent Events (SSE) streaming response Result.

    Args:
        stream_generator: A source yielding individual data items. This can be an
                          async generator, sync generator, iterable, or a single item.
                          Each item will be formatted as an SSE event.
        info: Optional help text for the Result.
        interface: Optional ToolBoxInterface to target.
        cleanup_func: Optional cleanup function to run when the stream ends or is cancelled.
        #http_headers: Optional dictionary of custom HTTP headers for the SSE response.

    Returns:
        A Result object configured for SSE streaming.
    """
    # Result.stream will handle calling SSEGenerator.create_sse_stream
    # and setting appropriate default headers for SSE when content_type is "text/event-stream".
    return cls.stream(
        stream_generator=stream_generator,
        content_type="text/event-stream",
        # headers=http_headers, # Pass if we add http_headers param
        info=info,
        interface=interface,
        cleanup_func=cleanup_func
    )

stream(stream_generator, content_type='text/event-stream', headers=None, info='OK', interface=ToolBoxInterfaces.remote, cleanup_func=None) classmethod

Create a streaming response Result. Handles SSE and other stream types.

Parameters:

Name	Type	Description	Default
stream_generator	Any	Any stream source (async generator, sync generator, iterable, or single item).	required
content_type	str	Content-Type header (default: text/event-stream for SSE).	'text/event-stream'
headers	dict | None	Additional HTTP headers for the response.	None
info	str	Help text for the result.	'OK'
interface	ToolBoxInterfaces	Interface to send data to.	remote
cleanup_func	Callable[[], None] | Callable[[], T] | Callable[[], AsyncGenerator[T, None]] | None	Optional function for cleanup.	None

Returns:

Type	Description
	A Result object configured for streaming.

Source code in toolboxv2/utils/system/types.py

@classmethod
def stream(cls,
           stream_generator: Any,  # Renamed from source for clarity
           content_type: str = "text/event-stream",  # Default to SSE
           headers: dict | None = None,
           info: str = "OK",
           interface: ToolBoxInterfaces = ToolBoxInterfaces.remote,
           cleanup_func: Callable[[], None] | Callable[[], T] | Callable[[], AsyncGenerator[T, None]] | None = None):
    """
    Create a streaming response Result. Handles SSE and other stream types.

    Args:
        stream_generator: Any stream source (async generator, sync generator, iterable, or single item).
        content_type: Content-Type header (default: text/event-stream for SSE).
        headers: Additional HTTP headers for the response.
        info: Help text for the result.
        interface: Interface to send data to.
        cleanup_func: Optional function for cleanup.

    Returns:
        A Result object configured for streaming.
    """
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=0, help_text=info)

    final_generator: AsyncGenerator[str, None]

    if content_type == "text/event-stream":
        # For SSE, always use SSEGenerator.create_sse_stream to wrap the source.
        # SSEGenerator.create_sse_stream handles various types of stream_generator internally.
        final_generator = SSEGenerator.create_sse_stream(source=stream_generator, cleanup_func=cleanup_func)

        # Standard SSE headers for the HTTP response itself
        # These will be stored in the Result object. Rust side decides how to use them.
        standard_sse_headers = {
            "Cache-Control": "no-cache",  # SSE specific
            "Connection": "keep-alive",  # SSE specific
            "X-Accel-Buffering": "no",  # Useful for proxies with SSE
            # Content-Type is implicitly text/event-stream, will be in streaming_data below
        }
        all_response_headers = standard_sse_headers.copy()
        if headers:
            all_response_headers.update(headers)
    else:
        # For non-SSE streams.
        # If stream_generator is sync, wrap it to be async.
        # If already async or single item, it will be handled.
        # Rust's stream_generator in ToolboxClient seems to handle both sync/async Python generators.
        # For consistency with how SSEGenerator does it, we can wrap sync ones.
        if inspect.isgenerator(stream_generator) or \
            (not isinstance(stream_generator, str) and hasattr(stream_generator, '__iter__')):
            final_generator = SSEGenerator.wrap_sync_generator(stream_generator)  # Simple async wrapper
        elif inspect.isasyncgen(stream_generator):
            final_generator = stream_generator
        else:  # Single item or string
            async def _single_item_gen():
                yield stream_generator

            final_generator = _single_item_gen()
        all_response_headers = headers if headers else {}

    # Prepare streaming data to be stored in the Result object
    streaming_data = {
        "type": "stream",  # Indicator for Rust side
        "generator": final_generator,
        "content_type": content_type,  # Let Rust know the intended content type
        "headers": all_response_headers  # Intended HTTP headers for the overall response
    }

    result_payload = ToolBoxResult(
        data_to=interface,
        data=streaming_data,
        data_info="Streaming response" if content_type != "text/event-stream" else "SSE Event Stream",
        data_type="stream"  # Generic type for Rust to identify it needs to stream from 'generator'
    )

    return cls(error=error, info=info_obj, result=result_payload)

text(text_data, content_type='text/plain', exec_code=None, status=200, info='OK', interface=ToolBoxInterfaces.remote, headers=None) classmethod

Create a text response Result with specific content type.

Source code in toolboxv2/utils/system/types.py

@classmethod
def text(cls, text_data, content_type="text/plain",exec_code=None,status=200, info="OK", interface=ToolBoxInterfaces.remote, headers=None):
    """Create a text response Result with specific content type."""
    if headers is not None:
        return cls.html(text_data, status= exec_code or status, info=info, headers=headers)
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=exec_code or status, help_text=info)

    result = ToolBoxResult(
        data_to=interface,
        data=text_data,
        data_info="Text response",
        data_type=content_type
    )

    return cls(error=error, info=info_obj, result=result)

typed_aget(key=None, default=None) async

Async get data with type validation

Source code in toolboxv2/utils/system/types.py

async def typed_aget(self, key=None, default=None) -> T:
    """Async get data with type validation"""
    data = await self.aget(key, default)

    if self._generic_type and data is not None:
        if not self._validate_type(data, self._generic_type):
            from toolboxv2 import get_logger
            get_logger().warning(f"Type mismatch: expected {self._generic_type}, got {type(data)}")

    return data

typed_get(key=None, default=None)

Get data with type validation

Source code in toolboxv2/utils/system/types.py

def typed_get(self, key=None, default=None) -> T:
    """Get data with type validation"""
    data = self.get(key, default)

    if self._generic_type and data is not None:
        # Validate type matches generic parameter
        if not self._validate_type(data, self._generic_type):
            from toolboxv2 import get_logger
            get_logger().warning(f"Type mismatch: expected {self._generic_type}, got {type(data)}")

    return data

typed_json(data, info='OK', interface=ToolBoxInterfaces.remote, exec_code=0, status_code=None) classmethod

Create JSON result with type information

Source code in toolboxv2/utils/system/types.py

@classmethod
def typed_json(cls, data: T, info="OK", interface=ToolBoxInterfaces.remote, exec_code=0,
               status_code=None) -> 'Result[T]':
    """Create JSON result with type information"""
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=status_code or exec_code, help_text=info)

    result = ToolBoxResult(
        data_to=interface,
        data=data,
        data_info="JSON response",
        data_type="json"
    )

    instance = cls(error=error, info=info_obj, result=result)
    if hasattr(cls, '_generic_type'):
        instance._generic_type = cls._generic_type

    return instance

typed_ok(data, data_info='', info='OK', interface=ToolBoxInterfaces.native) classmethod

Create OK result with type information

Source code in toolboxv2/utils/system/types.py

@classmethod
def typed_ok(cls, data: T, data_info="", info="OK", interface=ToolBoxInterfaces.native) -> 'Result[T]':
    """Create OK result with type information"""
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=0, help_text=info)
    result = ToolBoxResult(data_to=interface, data=data, data_info=data_info, data_type=type(data).__name__)

    instance = cls(error=error, info=info_obj, result=result)
    if hasattr(cls, '_generic_type'):
        instance._generic_type = cls._generic_type

    return instance

all_functions_enums

Automatic generated by ToolBox v = 0.1.22

main_tool

MainTool

Source code in toolboxv2/utils/system/main_tool.py

class MainTool:
    toolID: str = ""
    # app = None
    interface = None
    spec = "app"
    name = ""
    color = "Bold"
    stuf = False

    def __init__(self, *args, **kwargs):
        """
        Standard constructor used for arguments pass
        Do not override. Use __ainit__ instead
        """
        self.__storedargs = args, kwargs
        self.tools = kwargs.get("tool", {})
        self.logger = kwargs.get("logs", get_logger())
        self.color = kwargs.get("color", "WHITE")
        self.todo = kwargs.get("load", kwargs.get("on_start", lambda: None))
        if "on_exit" in kwargs and isinstance(kwargs.get("on_exit"), Callable):
            self.on_exit =self.app.tb(
                mod_name=self.name,
                name=kwargs.get("on_exit").__name__,
                version=self.version if hasattr(self, 'version') else "0.0.0",
            )(kwargs.get("on_exit"))
        self.async_initialized = False
        if self.todo:
            try:
                if inspect.iscoroutinefunction(self.todo):
                    pass
                else:
                    self.todo()
                get_logger().info(f"{self.name} on load suspended")
            except Exception as e:
                get_logger().error(f"Error loading mod {self.name} {e}")
                if self.app.debug:
                    import traceback
                    traceback.print_exc()
        else:
            get_logger().info(f"{self.name} no load require")

    async def __ainit__(self, *args, **kwargs):
        self.version = kwargs.get("v", kwargs.get("version", "0.0.0"))
        self.tools = kwargs.get("tool", {})
        self.name = kwargs["name"]
        self.logger = kwargs.get("logs", get_logger())
        self.color = kwargs.get("color", "WHITE")
        self.todo = kwargs.get("load", kwargs.get("on_start"))
        if not hasattr(self, 'config'):
            self.config = {}
        self.user = None
        self.description = "A toolbox mod" if kwargs.get("description") is None else kwargs.get("description")
        if MainTool.interface is None:
            MainTool.interface = self.app.interface_type
        # Result.default(self.app.interface)

        if self.todo:
            try:
                if inspect.iscoroutinefunction(self.todo):
                    await self.todo()
                else:
                    pass
                await asyncio.sleep(0.1)
                get_logger().info(f"{self.name} on load suspended")
            except Exception as e:
                get_logger().error(f"Error loading mod {self.name} {e}")
                if self.app.debug:
                    import traceback
                    traceback.print_exc()
        else:
            get_logger().info(f"{self.name} no load require")
        self.app.print(f"TOOL : {self.spec}.{self.name} online")



    @property
    def app(self):
        return get_app(
            from_=f"{self.spec}.{self.name}|{self.toolID if self.toolID else '*' + MainTool.toolID} {self.interface if self.interface else MainTool.interface}")

    @app.setter
    def app(self, v):
        raise PermissionError(f"You cannot set the App Instance! {v=}")

    @staticmethod
    def return_result(error: ToolBoxError = ToolBoxError.none,
                      exec_code: int = 0,
                      help_text: str = "",
                      data_info=None,
                      data=None,
                      data_to=None):

        if data_to is None:
            data_to = MainTool.interface if MainTool.interface is not None else ToolBoxInterfaces.cli

        if data is None:
            data = {}

        if data_info is None:
            data_info = {}

        return Result(
            error,
            ToolBoxResult(data_info=data_info, data=data, data_to=data_to),
            ToolBoxInfo(exec_code=exec_code, help_text=help_text)
        )

    def print(self, message, end="\n", **kwargs):
        if self.stuf:
            return

        self.app.print(Style.style_dic[self.color] + self.name + Style.style_dic["END"] + ":", message, end=end,
                       **kwargs)

    def add_str_to_config(self, command):
        if len(command) != 2:
            self.logger.error('Invalid command must be key value')
            return False
        self.config[command[0]] = command[1]

    def webInstall(self, user_instance, construct_render) -> str:
        """"Returns a web installer for the given user instance and construct render template"""

    def get_version(self) -> str:
        """"Returns the version"""
        return self.version

    async def get_user(self, username: str) -> Result:
        return await self.app.a_run_any(CLOUDM_AUTHMANAGER.GET_USER_BY_NAME, username=username, get_results=True)

    async def __initobj(self):
        """Crutch used for __await__ after spawning"""
        assert not self.async_initialized
        self.async_initialized = True
        # pass the parameters to __ainit__ that passed to __init__
        await self.__ainit__(*self.__storedargs[0], **self.__storedargs[1])
        return self

    def __await__(self):
        return self.__initobj().__await__()

__init__(*args, **kwargs)

Standard constructor used for arguments pass Do not override. Use ainit instead

Source code in toolboxv2/utils/system/main_tool.py

def __init__(self, *args, **kwargs):
    """
    Standard constructor used for arguments pass
    Do not override. Use __ainit__ instead
    """
    self.__storedargs = args, kwargs
    self.tools = kwargs.get("tool", {})
    self.logger = kwargs.get("logs", get_logger())
    self.color = kwargs.get("color", "WHITE")
    self.todo = kwargs.get("load", kwargs.get("on_start", lambda: None))
    if "on_exit" in kwargs and isinstance(kwargs.get("on_exit"), Callable):
        self.on_exit =self.app.tb(
            mod_name=self.name,
            name=kwargs.get("on_exit").__name__,
            version=self.version if hasattr(self, 'version') else "0.0.0",
        )(kwargs.get("on_exit"))
    self.async_initialized = False
    if self.todo:
        try:
            if inspect.iscoroutinefunction(self.todo):
                pass
            else:
                self.todo()
            get_logger().info(f"{self.name} on load suspended")
        except Exception as e:
            get_logger().error(f"Error loading mod {self.name} {e}")
            if self.app.debug:
                import traceback
                traceback.print_exc()
    else:
        get_logger().info(f"{self.name} no load require")

__initobj() async

Crutch used for await after spawning

Source code in toolboxv2/utils/system/main_tool.py

async def __initobj(self):
    """Crutch used for __await__ after spawning"""
    assert not self.async_initialized
    self.async_initialized = True
    # pass the parameters to __ainit__ that passed to __init__
    await self.__ainit__(*self.__storedargs[0], **self.__storedargs[1])
    return self

get_version()

"Returns the version

Source code in toolboxv2/utils/system/main_tool.py

def get_version(self) -> str:
    """"Returns the version"""
    return self.version

webInstall(user_instance, construct_render)

"Returns a web installer for the given user instance and construct render template

Source code in toolboxv2/utils/system/main_tool.py

def webInstall(self, user_instance, construct_render) -> str:
    """"Returns a web installer for the given user instance and construct render template"""

get_version_from_pyproject(pyproject_path='../pyproject.toml')

Reads the version from the pyproject.toml file.

Source code in toolboxv2/utils/system/main_tool.py

def get_version_from_pyproject(pyproject_path='../pyproject.toml'):
    """Reads the version from the pyproject.toml file."""
    if not os.path.exists(pyproject_path) and pyproject_path=='../pyproject.toml':
        pyproject_path = 'pyproject.toml'
    if not os.path.exists(pyproject_path) and pyproject_path=='pyproject.toml':
        return "0.1.21"

    try:
        import toml
        # Load the pyproject.toml file
        with open(pyproject_path) as file:
            pyproject_data = toml.load(file)

        # Extract the version from the 'project' section
        version = pyproject_data.get('project', {}).get('version')

        if version is None:
            raise ValueError(f"Version not found in {pyproject_path}")

        return version
    except Exception as e:
        print(f"Error reading version: {e}")
        return "0.0.0"

session

Session

Source code in toolboxv2/utils/system/session.py

class Session(metaclass=Singleton):

    # user: LocalUser

    def __init__(self, username, base=None):
        self.username = username
        self._session: ClientSession | None = None
        self._event_loop = None  # Track which event loop the session belongs to
        self.valid = False
        if base is None:
            base = os.environ.get("TOOLBOXV2_REMOTE_BASE", "https://simplecore.app")
        if base is not None and base.endswith("/api/"):
            base = base.replace("api/", "")
        self.base = base
        self.base = base.rstrip('/')  # Ensure no trailing slash

    @property
    def session(self):
        self._ensure_session()
        return self._session

    def _ensure_session(self):
        """Ensure session is valid for current event loop"""
        try:
            current_loop = asyncio.get_running_loop()
        except RuntimeError:
            # No running loop
            if self._session is not None:
                # Close old session if it exists
                self._session = None
                self._event_loop = None
            return

        # Check if session exists and is for the current loop
        if self._session is None or self._event_loop != current_loop:
            # Close old session if it exists and is from a different loop
            if self._session is not None:
                try:
                    # Try to close old session, but don't fail if loop is closed
                    if not self._session.closed:
                        asyncio.create_task(self._session.close())
                except:
                    pass
            # Create new session for current loop
            self._session = ClientSession()
            self._event_loop = current_loop

    async def init_log_in_mk_link(self, mak_link):
        from urllib.parse import parse_qs, urlparse
        await asyncio.sleep(0.1)

        pub_key, prv_key = Code.generate_asymmetric_keys()
        Code.save_keys_to_files(pub_key, prv_key, get_app('sys.session').info_dir.replace(get_app('sys.session').id, ''))
        parsed_url = urlparse(mak_link)
        params = parse_qs(parsed_url.query)
        invitation = params.get('key', [None])[0]
        self.username = params.get('name', [get_app('sys.session').get_username()])[0]
        if not invitation:
            print('Invalid LoginKey')
            return False

        res = await get_app("Session.InitLogin").run_http("CloudM.AuthManager", "add_user_device", method="POST",
                                                          name=self.username, pub_key=pub_key, invitation=invitation,
                                                          web_data=False, as_base64=False)
        res = Result.result_from_dict(**res)
        if res.is_error():
            return res
        await asyncio.sleep(0.1)
        return await self.auth_with_prv_key()

    @staticmethod
    def get_prv_key():
        pub_key, prv_key = Code.load_keys_from_files(get_app('sys.session.get_prv_key').info_dir.replace(get_app('sys.session.get_prv_key').id, ''))
        return prv_key

    def if_key(self):
        return len(self.get_prv_key()) > 0

    async def auth_with_prv_key(self):
        prv_key = self.get_prv_key()
        if not prv_key:
            return False
        challenge = await get_app("Session.InitLogin").run_http('CloudM.AuthManager', 'get_to_sing_data', method="POST",
                                                                args_='username=' + self.username + '&personal_key=False')
        challenge = Result.result_from_dict(**await challenge)
        if challenge.is_error():
            return challenge.lazy_return(-1, data=challenge.error)

        await asyncio.sleep(0.1)
        claim_data = await get_app("Session.InitLogin").run_http('CloudM.AuthManager', 'validate_device',
                                                                 username=self.username,
                                                                 signature=Code.create_signature(
                                                                     challenge.get("challenge"), prv_key,
                                                                     salt_length=32),
                                                                 method="POST")
        claim_data = Result.result_from_dict(**claim_data)
        if claim_data.is_error():
            return claim_data.lazy_return(-1, data=claim_data.error)

        claim = claim_data.get("key")

        if claim is None:
            return claim_data
        await asyncio.sleep(0.1)
        with BlobFile(f"claim/{self.username}/jwt.c", key=Code.DK()(), mode="w") as blob:
            blob.clear()
            blob.write(claim.encode())
        await asyncio.sleep(0.1)
        # Do something with the data or perform further actions
        res = await self.login()
        return res

    def init(self):
        self._ensure_session()

    async def login(self, verbose=False):
        self._ensure_session()
        with BlobFile(f"claim/{self.username}/jwt.c", key=Code.DK()(), mode="r") as blob:
            claim = blob.read()
            if claim == b'Error decoding':
                blob.clear()
                claim = b''
        if not claim:
            res = await self.auth_with_prv_key()
            return res is True
        try:
            self._ensure_session()  # Ensure session is valid before using
            async with self.session.request("POST", url=f"{self.base}/validateSession", json={'Jwt_claim': claim.decode(),
                                                                                             'Username': self.username}) as response:
                # print(response.status, "status")
                if response.status == 200:
                    print("Successfully Connected 2 TBxN")
                    get_logger().info("LogIn successful")
                    self.valid = True
                    return True
                if response.status == 401 and self.if_key():
                    return await self.auth_with_prv_key()
                # print(response)
                get_logger().warning("LogIn failed")
                return False
        except ClientConnectorError as e:
            print(f"Server nicht erreichbar (DNS oder Verbindung): {e}") if verbose else None
            return False
        except socket.gaierror as e:
            print(f"DNS-Auflösung fehlgeschlagen: {e}") if verbose else None
            return False
        except ClientError as e:
            print(f"Allgemeiner Client-Fehler: {e}") if verbose else None
            return False
        except Exception as e:
            print("Connection error", self.username, e) if verbose else None
            return False

    async def download_file(self, url, dest_folder="mods_sto"):
        self._ensure_session()
        if not self.session:
            raise Exception("Session not initialized. Please login first.")
        # Sicherstellen, dass das Zielverzeichnis existiert
        os.makedirs(dest_folder, exist_ok=True)

        # Analyse der URL, um den Dateinamen zu extrahieren
        filename = url.split('/')[-1]

        # Bereinigen des Dateinamens von Sonderzeichen
        valid_chars = '-_.()abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'
        filename = ''.join(char for char in filename if char in valid_chars)

        # Konstruieren des vollständigen Dateipfads
        file_path = os.path.join(dest_folder, filename)
        if isinstance(url, str):
            url = self.base + url
        try:
            async with self.session.get(url) as response:
                if response.status == 200:
                    with open(file_path, 'wb') as f:
                        while True:
                            chunk = await response.content.read(1024)
                            if not chunk:
                                break
                            f.write(chunk)
                    print(f'File downloaded: {file_path}')
                    return True
                else:
                    print(f'Failed to download file: {url}. Status code: {response.status}')
        except ClientConnectorError as e:
            print(f"Server nicht erreichbar (DNS oder Verbindung): {e}")
            return False
        except socket.gaierror as e:
            print(f"DNS-Auflösung fehlgeschlagen: {e}")
            return False
        except ClientError as e:
            print(f"Allgemeiner Client-Fehler: {e}")
            return False
        except Exception as e:
            print("Error:",e, self.username)
        return False

    async def logout(self) -> bool:
        self._ensure_session()
        if self.session and not self.session.closed:  # Sicherstellen, dass die Session offen ist
            try:
                # Der `post`-Aufruf gibt eine Coroutine zurück, die wir awaiten müssen,
                # um das Response-Objekt zu erhalten.
                response = await self.session.post(f'{self.base}/web/logoutS')

                # Wir verwenden `async with` für die Response, um sicherzustellen, dass sie richtig behandelt wird.
                async with response:
                    is_successful = response.status == 200

                # Die Session erst NACH der Anfrage schließen.
                await self.session.close()
                self.session = None
                self._event_loop = None

                return is_successful
            except (ClientConnectorError, socket.gaierror, ClientError) as e:
                # Zusammengefasste Fehlerbehandlung für Verbindungsfehler
                print(f"Fehler bei der Verbindung während des Logouts: {e}")
                if self.session:
                    await self.session.close()
                self.session = None
                self._event_loop = None
                return False
            except Exception as e:
                print(f"Allgemeiner Fehler während des Logouts: {e}, user: {self.username}")
                if self.session:
                    await self.session.close()
                self.session = None
                self._event_loop = None
        return False  # Gibt False zurück, wenn keine Session vorhanden war

    async def fetch(self, url: str, method: str = 'GET', data=None, **kwargs) -> bool | ClientResponse | Response:
        # Ensure we have a valid session for the current event loop
        self._ensure_session()

        if isinstance(url, str):
            url = self.base + url
        if self.session:
            try:
                if method.upper() == 'POST':
                    return await self.session.post(url, json=data, **kwargs)
                else:
                    return await self.session.get(url, **kwargs)
            except ClientConnectorError as e:
                print(f"Server nicht erreichbar (DNS oder Verbindung): {e}")
                return False
            except socket.gaierror as e:
                print(f"DNS-Auflösung fehlgeschlagen: {e}")
                return False
            except ClientError as e:
                print(f"Allgemeiner Client-Fehler: {e}")
                return False
            except Exception as e:
                print("Error session fetch:", e, self.username)
                import traceback
                print(traceback.format_exc())
                return requests.request(method, url, data=data)
        else:
            print(f"Could not find session using request on {url}")
            if method.upper() == 'POST':
                return requests.request(method, url, json=data)
            return requests.request(method, url, data=data)
            # raise Exception("Session not initialized. Please login first.")

    async def upload_file(self, file_path: str, upload_url: str):
        # Prüfe, ob die Datei existiert
        if not os.path.isfile(file_path):
            raise FileNotFoundError(f"Datei {file_path} nicht gefunden.")

        # Initialisiere die Session, falls sie nicht bereits gestartet ist
        self._ensure_session()
        upload_url = self.base + upload_url
        # headers = {'accept': '*/*',
        #            'Content-Type': 'multipart/form-data'}
        # file_dict = {"file": open(file_path, "rb")}
        # response = await self.session.post(upload_url, headers=headers, data=file_dict)
        with open(file_path, 'rb') as f:
            file_data = f.read()

        with MultipartWriter('form-data') as mpwriter:
            part = mpwriter.append(file_data)
            part.set_content_disposition('form-data', name='file', filename=os.path.basename(file_path))
            try:
                async with self.session.post(upload_url, data=mpwriter, timeout=20000) as response:

                    # Prüfe, ob der Upload erfolgreich war
                    if response.status == 200:
                        print(f"Datei {file_path} erfolgreich hochgeladen.")
                        return await response.json()
                    else:
                        print(f"Fehler beim Hochladen der Datei {file_path}. Status: {response.status}")
                        print(await response.text())
                        return None
            except ClientConnectorError as e:
                print(f"Server nicht erreichbar (DNS oder Verbindung): {e}")
                return False
            except socket.gaierror as e:
                print(f"DNS-Auflösung fehlgeschlagen: {e}")
                return False
            except ClientError as e:
                print(f"Allgemeiner Client-Fehler: {e}")
                return False
            except Exception as e:
                print(f"Error session Fehler beim Hochladen der Datei {file_path}:", e, self.username)



    async def cleanup(self):
        """Cleanup session resources - should be called before application shutdown"""
        try:
            if self._session is not None and not self._session.closed:
                await self._session.close()
        except ClientConnectorError as e:
            print(f"Server nicht erreichbar (DNS oder Verbindung): {e}")
        except socket.gaierror as e:
            print(f"DNS-Auflösung fehlgeschlagen: {e}")
        except ClientError as e:
            print(f"Allgemeiner Client-Fehler: {e}")
        except Exception as e:
            pass
        finally:
            self._session = None
            self._event_loop = None

    def exit(self):
        with BlobFile(f"claim/{self.username}/jwt.c", key=Code.DK()(), mode="w") as blob:
            blob.clear()

cleanup() async

Cleanup session resources - should be called before application shutdown

Source code in toolboxv2/utils/system/session.py

async def cleanup(self):
    """Cleanup session resources - should be called before application shutdown"""
    try:
        if self._session is not None and not self._session.closed:
            await self._session.close()
    except ClientConnectorError as e:
        print(f"Server nicht erreichbar (DNS oder Verbindung): {e}")
    except socket.gaierror as e:
        print(f"DNS-Auflösung fehlgeschlagen: {e}")
    except ClientError as e:
        print(f"Allgemeiner Client-Fehler: {e}")
    except Exception as e:
        pass
    finally:
        self._session = None
        self._event_loop = None

state_system

The Task of the State System is : 1 Kep trak of the current state of the ToolBox and its dependency's 2 tracks the shasum of all mod and runnabael 3 the version of all mod

The state : {"utils":{"file_name": {"version":##,"shasum"}} ,"mods":{"file_name": {"version":##,"shasum":##,"src-url":##}} ,"runnable":{"file_name": {"version":##,"shasum":##,"src-url":##}} ,"api":{"file_name": {"version":##,"shasum"}} ,"app":{"file_name": {"version":##,"shasum":##,"src-url":##}} }

trans form state from on to an other.

detect_os_and_arch()

Detect the current operating system and architecture.

Source code in toolboxv2/utils/system/state_system.py

def detect_os_and_arch():
    """Detect the current operating system and architecture."""
    current_os = platform.system().lower()  # e.g., 'windows', 'linux', 'darwin'
    machine = platform.machine().lower()  # e.g., 'x86_64', 'amd64'
    return current_os, machine

download_executable(url, file_name)

Attempt to download the executable from the provided URL.

Source code in toolboxv2/utils/system/state_system.py

def download_executable(url, file_name):
    """Attempt to download the executable from the provided URL."""
    try:
        import requests
    except ImportError:
        print("The 'requests' library is required. Please install it via pip install requests")
        sys.exit(1)

    print(f"Attempting to download executable from {url}...")
    try:
        response = requests.get(url, stream=True)
    except Exception as e:
        print(f"Download error: {e}")
        return None

    if response.status_code == 200:
        with open(file_name, "wb") as f:
            for chunk in response.iter_content(chunk_size=8192):
                if chunk:
                    f.write(chunk)
        # Make the file executable on non-Windows systems
        if platform.system().lower() != "windows":
            os.chmod(file_name, 0o755)
        return file_name
    else:
        print("Download failed. Status code:", response.status_code)
        return None

find_highest_zip_version(name_filter, app_version=None, root_dir='mods_sto', version_only=False)

Findet die höchste verfügbare ZIP-Version in einem Verzeichnis basierend auf einem Namensfilter.

Parameters:

Name	Type	Description	Default
root_dir	str	Wurzelverzeichnis für die Suche	'mods_sto'
name_filter	str	Namensfilter für die ZIP-Dateien	required
app_version	str	Aktuelle App-Version für Kompatibilitätsprüfung	None

Returns:

Name	Type	Description
str	str	Pfad zur ZIP-Datei mit der höchsten Version oder None wenn keine gefunden

Source code in toolboxv2/utils/system/state_system.py

def find_highest_zip_version(name_filter: str, app_version: str = None, root_dir: str = "mods_sto", version_only=False) -> str:
    """
    Findet die höchste verfügbare ZIP-Version in einem Verzeichnis basierend auf einem Namensfilter.

    Args:
        root_dir (str): Wurzelverzeichnis für die Suche
        name_filter (str): Namensfilter für die ZIP-Dateien
        app_version (str, optional): Aktuelle App-Version für Kompatibilitätsprüfung

    Returns:
        str: Pfad zur ZIP-Datei mit der höchsten Version oder None wenn keine gefunden
    """

    from packaging import version

    # Kompiliere den Regex-Pattern für die Dateinamen
    pattern = fr"{name_filter}&v[0-9.]+§([0-9.]+)\.zip$"

    highest_version = None
    highest_version_file = None

    # Durchsuche das Verzeichnis
    root_path = Path(root_dir)
    for file_path in root_path.rglob("*.zip"):
        if "RST$"+name_filter not in str(file_path):
            continue
        match = re.search(pattern, str(file_path).split("RST$")[-1].strip())
        if match:
            zip_version = match.group(1)

            # Prüfe App-Version Kompatibilität falls angegeben
            if app_version:
                file_app_version = re.search(r"&v([0-9.]+)§", str(file_path)).group(1)
                if version.parse(file_app_version) > version.parse(app_version):
                    continue

            # Vergleiche Versionen
            current_version = version.parse(zip_version)
            if highest_version is None or current_version > highest_version:
                highest_version = current_version
                highest_version_file = str(file_path)
    if version_only:
        return str(highest_version)
    return highest_version_file

find_highest_zip_version_entry(name, target_app_version=None, filepath='tbState.yaml')

Findet den Eintrag mit der höchsten ZIP-Version für einen gegebenen Namen und eine optionale Ziel-App-Version in einer YAML-Datei.

:param name: Der Name des gesuchten Eintrags. :param target_app_version: Die Zielversion der App als String (optional). :param filepath: Der Pfad zur YAML-Datei. :return: Den Eintrag mit der höchsten ZIP-Version innerhalb der Ziel-App-Version oder None, falls nicht gefunden.

Source code in toolboxv2/utils/system/state_system.py

def find_highest_zip_version_entry(name, target_app_version=None, filepath='tbState.yaml'):
    """
    Findet den Eintrag mit der höchsten ZIP-Version für einen gegebenen Namen und eine optionale Ziel-App-Version in einer YAML-Datei.

    :param name: Der Name des gesuchten Eintrags.
    :param target_app_version: Die Zielversion der App als String (optional).
    :param filepath: Der Pfad zur YAML-Datei.
    :return: Den Eintrag mit der höchsten ZIP-Version innerhalb der Ziel-App-Version oder None, falls nicht gefunden.
    """
    import yaml

    from packaging import version

    highest_zip_ver = None
    highest_entry = {}

    with open(filepath) as file:
        data = yaml.safe_load(file)
        # print(data)
        app_ver_h = None
        for key, value in list(data.get('installable', {}).items())[::-1]:
            # Prüfe, ob der Name im Schlüssel enthalten ist

            if name in key:
                v = value['version']
                if len(v) == 1:
                    app_ver = v[0].split('v')[-1]
                    zip_ver = "0.0.0"
                else:
                    app_ver, zip_ver = v
                    app_ver = app_ver.split('v')[-1]
                app_ver = version.parse(app_ver)
                # Wenn eine Ziel-App-Version angegeben ist, vergleiche sie
                if target_app_version is None or app_ver == version.parse(target_app_version):
                    current_zip_ver = version.parse(zip_ver)
                    # print(current_zip_ver, highest_zip_ver)

                    if highest_zip_ver is None or current_zip_ver > highest_zip_ver:
                        highest_zip_ver = current_zip_ver
                        highest_entry = value

                    if app_ver_h is None or app_ver > app_ver_h:
                        app_ver_h = app_ver
                        highest_zip_ver = current_zip_ver
                        highest_entry = value
    return highest_entry

query_executable_url(current_os, machine)

Query a remote URL for a matching executable based on OS and architecture. The file name is built dynamically based on parameters.

Source code in toolboxv2/utils/system/state_system.py

def query_executable_url(current_os, machine):
    """
    Query a remote URL for a matching executable based on OS and architecture.
    The file name is built dynamically based on parameters.
    """
    base_url = "https://example.com/downloads"  # Replace with the actual URL
    # Windows executables have .exe extension
    if current_os == "windows":
        file_name = f"server_{current_os}_{machine}.exe"
    else:
        file_name = f"server_{current_os}_{machine}"
    full_url = f"{base_url}/{file_name}"
    return full_url, file_name

types

AppType

Source code in toolboxv2/utils/system/types.py

class AppType:
    prefix: str
    id: str
    globals: dict[str, Any] = {"root": dict, }
    locals: dict[str, Any] = {"user": {'app': "self"}, }

    local_test: bool = False
    start_dir: str
    data_dir: str
    config_dir: str
    info_dir: str
    is_server:bool = False

    logger: logging.Logger
    logging_filename: str

    api_allowed_mods_list: list[str] = []

    version: str
    loop: asyncio.AbstractEventLoop

    keys: dict[str, str] = {
        "MACRO": "macro~~~~:",
        "MACRO_C": "m_color~~:",
        "HELPER": "helper~~~:",
        "debug": "debug~~~~:",
        "id": "name-spa~:",
        "st-load": "mute~load:",
        "comm-his": "comm-his~:",
        "develop-mode": "dev~mode~:",
        "provider::": "provider::",
    }

    defaults: dict[str, (bool or dict or dict[str, dict[str, str]] or str or list[str] or list[list]) | None] = {
        "MACRO": list[str],
        "MACRO_C": dict,
        "HELPER": dict,
        "debug": str,
        "id": str,
        "st-load": False,
        "comm-his": list[list],
        "develop-mode": bool,
    }

    cluster_manager: ClusterManager
    root_blob_storage: BlobStorage
    config_fh: FileHandler
    _debug: bool
    flows: dict[str, Callable]
    dev_modi: bool
    functions: dict[str, Any]
    modules: dict[str, Any]

    interface_type: ToolBoxInterfaces
    REFIX: str

    alive: bool
    called_exit: tuple[bool, float]
    args_sto: AppArgs
    system_flag = None
    session = None
    appdata = None
    exit_tasks = []

    enable_profiling: bool = False
    sto = None

    websocket_handlers: dict[str, dict[str, Callable]] = {}
    _rust_ws_bridge: Any = None

    docs_reader: Callable | None = None
    docs_writer: Callable | None = None
    get_update_suggestions: Callable | None = None
    auto_update_docs: Callable | None = None
    source_code_lookup: Callable | None = None

    initial_docs_parse: Callable | None = None

    def __init__(self, prefix=None, args=None):
        self.args_sto = args
        self.prefix = prefix
        self._footprint_start_time = time.time()
        self._process = psutil.Process(os.getpid())

        # Tracking-Daten für Min/Max/Avg
        self._footprint_metrics = {
            'memory': {'max': 0, 'min': float('inf'), 'samples': []},
            'cpu': {'max': 0, 'min': float('inf'), 'samples': []},
            'disk_read': {'max': 0, 'min': float('inf'), 'samples': []},
            'disk_write': {'max': 0, 'min': float('inf'), 'samples': []},
            'network_sent': {'max': 0, 'min': float('inf'), 'samples': []},
            'network_recv': {'max': 0, 'min': float('inf'), 'samples': []},
        }

        # Initial Disk/Network Counters
        try:
            io_counters = self._process.io_counters()
            self._initial_disk_read = io_counters.read_bytes
            self._initial_disk_write = io_counters.write_bytes
        except (AttributeError, OSError):
            self._initial_disk_read = 0
            self._initial_disk_write = 0

        try:
            net_io = psutil.net_io_counters()
            self._initial_network_sent = net_io.bytes_sent
            self._initial_network_recv = net_io.bytes_recv
        except (AttributeError, OSError):
            self._initial_network_sent = 0
            self._initial_network_recv = 0

    def _update_metric_tracking(self, metric_name: str, value: float):
        """Aktualisiert Min/Max/Avg für eine Metrik"""
        metrics = self._footprint_metrics[metric_name]
        metrics['max'] = max(metrics['max'], value)
        metrics['min'] = min(metrics['min'], value)
        metrics['samples'].append(value)

        # Begrenze die Anzahl der Samples (letzte 1000)
        if len(metrics['samples']) > 1000:
            metrics['samples'] = metrics['samples'][-1000:]

    def _get_metric_avg(self, metric_name: str) -> float:
        """Berechnet Durchschnitt einer Metrik"""
        samples = self._footprint_metrics[metric_name]['samples']
        return sum(samples) / len(samples) if samples else 0

    def footprint(self, update_tracking: bool = True) -> FootprintMetrics:
        """
        Erfasst den aktuellen Ressourcen-Footprint der Toolbox-Instanz.

        Args:
            update_tracking: Wenn True, aktualisiert Min/Max/Avg-Tracking

        Returns:
            FootprintMetrics mit allen erfassten Metriken
        """
        current_time = time.time()
        uptime_seconds = current_time - self._footprint_start_time

        # Formatierte Uptime
        uptime_delta = timedelta(seconds=int(uptime_seconds))
        uptime_formatted = str(uptime_delta)

        # Memory Metrics (in MB)
        try:
            mem_info = self._process.memory_info()
            memory_current = mem_info.rss / (1024 * 1024)  # Bytes zu MB
            memory_percent = self._process.memory_percent()

            if update_tracking:
                self._update_metric_tracking('memory', memory_current)

            memory_max = self._footprint_metrics['memory']['max']
            memory_min = self._footprint_metrics['memory']['min']
            if memory_min == float('inf'):
                memory_min = memory_current
        except (psutil.NoSuchProcess, psutil.AccessDenied):
            memory_current = memory_max = memory_min = memory_percent = 0

        # CPU Metrics
        try:
            cpu_percent_current = self._process.cpu_percent(interval=0.1)
            cpu_times = self._process.cpu_times()
            cpu_time_seconds = cpu_times.user + cpu_times.system

            if update_tracking:
                self._update_metric_tracking('cpu', cpu_percent_current)

            cpu_percent_max = self._footprint_metrics['cpu']['max']
            cpu_percent_min = self._footprint_metrics['cpu']['min']
            cpu_percent_avg = self._get_metric_avg('cpu')

            if cpu_percent_min == float('inf'):
                cpu_percent_min = cpu_percent_current
        except (psutil.NoSuchProcess, psutil.AccessDenied):
            cpu_percent_current = cpu_percent_max = 0
            cpu_percent_min = cpu_percent_avg = cpu_time_seconds = 0

        # Disk I/O Metrics (in MB)
        try:
            io_counters = self._process.io_counters()
            disk_read_bytes = io_counters.read_bytes - self._initial_disk_read
            disk_write_bytes = io_counters.write_bytes - self._initial_disk_write

            disk_read_mb = disk_read_bytes / (1024 * 1024)
            disk_write_mb = disk_write_bytes / (1024 * 1024)

            if update_tracking:
                self._update_metric_tracking('disk_read', disk_read_mb)
                self._update_metric_tracking('disk_write', disk_write_mb)

            disk_read_max = self._footprint_metrics['disk_read']['max']
            disk_read_min = self._footprint_metrics['disk_read']['min']
            disk_write_max = self._footprint_metrics['disk_write']['max']
            disk_write_min = self._footprint_metrics['disk_write']['min']

            if disk_read_min == float('inf'):
                disk_read_min = disk_read_mb
            if disk_write_min == float('inf'):
                disk_write_min = disk_write_mb
        except (AttributeError, OSError, psutil.NoSuchProcess, psutil.AccessDenied):
            disk_read_mb = disk_write_mb = 0
            disk_read_max = disk_read_min = disk_write_max = disk_write_min = 0

        # Network I/O Metrics (in MB)
        try:
            net_io = psutil.net_io_counters()
            network_sent_bytes = net_io.bytes_sent - self._initial_network_sent
            network_recv_bytes = net_io.bytes_recv - self._initial_network_recv

            network_sent_mb = network_sent_bytes / (1024 * 1024)
            network_recv_mb = network_recv_bytes / (1024 * 1024)

            if update_tracking:
                self._update_metric_tracking('network_sent', network_sent_mb)
                self._update_metric_tracking('network_recv', network_recv_mb)

            network_sent_max = self._footprint_metrics['network_sent']['max']
            network_sent_min = self._footprint_metrics['network_sent']['min']
            network_recv_max = self._footprint_metrics['network_recv']['max']
            network_recv_min = self._footprint_metrics['network_recv']['min']

            if network_sent_min == float('inf'):
                network_sent_min = network_sent_mb
            if network_recv_min == float('inf'):
                network_recv_min = network_recv_mb
        except (AttributeError, OSError):
            network_sent_mb = network_recv_mb = 0
            network_sent_max = network_sent_min = 0
            network_recv_max = network_recv_min = 0

        # Process Info
        try:
            process_id = self._process.pid
            threads = self._process.num_threads()
            open_files_path = [str(x.path).replace("\\", "/") for x in self._process.open_files()]
            connections_uri = [f"{x.laddr}:{x.raddr} {str(x.status)}" for x in self._process.connections()]

            open_files = len(open_files_path)
            connections = len(connections_uri)
        except (psutil.NoSuchProcess, psutil.AccessDenied):
            process_id = os.getpid()
            threads = open_files = connections = 0
            open_files_path = []
            connections_uri = []

        return FootprintMetrics(
            start_time=self._footprint_start_time,
            uptime_seconds=uptime_seconds,
            uptime_formatted=uptime_formatted,
            memory_current=memory_current,
            memory_max=memory_max,
            memory_min=memory_min,
            memory_percent=memory_percent,
            cpu_percent_current=cpu_percent_current,
            cpu_percent_max=cpu_percent_max,
            cpu_percent_min=cpu_percent_min,
            cpu_percent_avg=cpu_percent_avg,
            cpu_time_seconds=cpu_time_seconds,
            disk_read_mb=disk_read_mb,
            disk_write_mb=disk_write_mb,
            disk_read_max=disk_read_max,
            disk_read_min=disk_read_min,
            disk_write_max=disk_write_max,
            disk_write_min=disk_write_min,
            network_sent_mb=network_sent_mb,
            network_recv_mb=network_recv_mb,
            network_sent_max=network_sent_max,
            network_sent_min=network_sent_min,
            network_recv_max=network_recv_max,
            network_recv_min=network_recv_min,
            process_id=process_id,
            threads=threads,
            open_files=open_files,
            connections=connections,
            open_files_path=open_files_path,
            connections_uri=connections_uri,
        )

    def print_footprint(self, detailed: bool = True) -> str:
        """
        Gibt den Footprint formatiert aus.

        Args:
            detailed: Wenn True, zeigt alle Details, sonst nur Zusammenfassung

        Returns:
            Formatierter Footprint-String
        """
        metrics = self.footprint()

        output = [
            "=" * 70,
            f"TOOLBOX FOOTPRINT - {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}",
            "=" * 70,
            f"\n📊 UPTIME",
            f"  Runtime: {metrics.uptime_formatted}",
            f"  Seconds: {metrics.uptime_seconds:.2f}s",
            f"\n💾 MEMORY USAGE",
            f"  Current:  {metrics.memory_current:.2f} MB ({metrics.memory_percent:.2f}%)",
            f"  Maximum:  {metrics.memory_max:.2f} MB",
            f"  Minimum:  {metrics.memory_min:.2f} MB",
        ]

        if detailed:
            helper_ = '\n\t- '.join(metrics.open_files_path)
            helper__ = '\n\t- '.join(metrics.connections_uri)
            output.extend([
                f"\n⚙️  CPU USAGE",
                f"  Current:  {metrics.cpu_percent_current:.2f}%",
                f"  Maximum:  {metrics.cpu_percent_max:.2f}%",
                f"  Minimum:  {metrics.cpu_percent_min:.2f}%",
                f"  Average:  {metrics.cpu_percent_avg:.2f}%",
                f"  CPU Time: {metrics.cpu_time_seconds:.2f}s",
                f"\n💿 DISK I/O",
                f"  Read:     {metrics.disk_read_mb:.2f} MB (Max: {metrics.disk_read_max:.2f}, Min: {metrics.disk_read_min:.2f})",
                f"  Write:    {metrics.disk_write_mb:.2f} MB (Max: {metrics.disk_write_max:.2f}, Min: {metrics.disk_write_min:.2f})",
                f"\n🌐 NETWORK I/O",
                f"  Sent:     {metrics.network_sent_mb:.2f} MB (Max: {metrics.network_sent_max:.2f}, Min: {metrics.network_sent_min:.2f})",
                f"  Received: {metrics.network_recv_mb:.2f} MB (Max: {metrics.network_recv_max:.2f}, Min: {metrics.network_recv_min:.2f})",
                f"\n🔧 PROCESS INFO",
                f"  PID:         {metrics.process_id}",
                f"  Threads:     {metrics.threads}",
                f"\n📂 OPEN FILES",
                f"  Open Files:  {metrics.open_files}",
                f"  Open Files Path: \n\t- {helper_}",
                f"\n🔗 NETWORK CONNECTIONS",
                f"  Connections: {metrics.connections}",
                f"  Connections URI: \n\t- {helper__}",
            ])

        output.append("=" * 70)

        return "\n".join(output)



    def start_server(self):
        from toolboxv2.utils.clis.api import manage_server
        if self.is_server:
            return
        manage_server("start")
        self.is_server = False

    @staticmethod
    def exit_main(*args, **kwargs):
        """proxi attr"""

    @staticmethod
    async def hide_console(*args, **kwargs):
        """proxi attr"""

    @staticmethod
    async def show_console(*args, **kwargs):
        """proxi attr"""

    @staticmethod
    async def disconnect(*args, **kwargs):
        """proxi attr"""

    def set_logger(self, debug=False):
        """proxi attr"""

    @property
    def debug(self):
        """proxi attr"""
        return self._debug

    def debug_rains(self, e):
        """proxi attr"""

    def set_flows(self, r):
        """proxi attr"""

    async def run_flows(self, name, **kwargs):
        """proxi attr"""

    def rrun_flows(self, name, **kwargs):
        """proxi attr"""

    def idle(self):
        import time
        self.print("idle")
        try:
            while self.alive:
                time.sleep(1)
        except KeyboardInterrupt:
            pass
        self.print("idle done")

    async def a_idle(self):
        self.print("a idle (running :"+("online)" if hasattr(self, 'daemon_app') else "offline)"))
        try:
            if hasattr(self, 'daemon_app'):
                await self.daemon_app.connect(self)
            else:
                while self.alive:
                    await asyncio.sleep(1)
        except KeyboardInterrupt:
            pass
        self.print("a idle done")

    @debug.setter
    def debug(self, value):
        """proxi attr"""

    def _coppy_mod(self, content, new_mod_dir, mod_name, file_type='py'):
        """proxi attr"""

    def _pre_lib_mod(self, mod_name, path_to="./runtime", file_type='py'):
        """proxi attr"""

    def _copy_load(self, mod_name, file_type='py', **kwargs):
        """proxi attr"""

    def inplace_load_instance(self, mod_name, loc="toolboxv2.mods.", spec='app', save=True):
        """proxi attr"""

    def save_instance(self, instance, modular_id, spec='app', instance_type="file/application", tools_class=None):
        """proxi attr"""

    def save_initialized_module(self, tools_class, spec):
        """proxi attr"""

    def mod_online(self, mod_name, installed=False):
        """proxi attr"""

    def _get_function(self,
                      name: Enum or None,
                      state: bool = True,
                      specification: str = "app",
                      metadata=False, as_str: tuple or None = None, r=0):
        """proxi attr"""

    def save_exit(self):
        """proxi attr"""

    def load_mod(self, mod_name: str, mlm='I', **kwargs):
        """proxi attr"""

    async def init_module(self, modular):
        return await self.load_mod(modular)

    async def load_external_mods(self):
        """proxi attr"""

    async def load_all_mods_in_file(self, working_dir="mods"):
        """proxi attr"""

    def get_all_mods(self, working_dir="mods", path_to="./runtime"):
        """proxi attr"""

    def remove_all_modules(self, delete=False):
        for mod in list(self.functions.keys()):
            self.logger.info(f"closing: {mod}")
            self.remove_mod(mod, delete=delete)

    async def a_remove_all_modules(self, delete=False):
        for mod in list(self.functions.keys()):
            self.logger.info(f"closing: {mod}")
            await self.a_remove_mod(mod, delete=delete)

    def print_ok(self):
        """proxi attr"""
        self.logger.info("OK")

    def reload_mod(self, mod_name, spec='app', is_file=True, loc="toolboxv2.mods."):
        """proxi attr"""

    def watch_mod(self, mod_name, spec='app', loc="toolboxv2.mods.", use_thread=True, path_name=None):
        """proxi attr"""

    def remove_mod(self, mod_name, spec='app', delete=True):
        """proxi attr"""

    async def a_remove_mod(self, mod_name, spec='app', delete=True):
        """proxi attr"""

    def exit(self):
        """proxi attr"""

    def web_context(self) -> str:
        """returns the build index ( toolbox web component )"""

    async def a_exit(self):
        """proxi attr"""

    def save_load(self, modname, spec='app'):
        """proxi attr"""

    def get_function(self, name: Enum or tuple, **kwargs):
        """
        Kwargs for _get_function
            metadata:: return the registered function dictionary
                stateless: (function_data, None), 0
                stateful: (function_data, higher_order_function), 0
            state::boolean
                specification::str default app
        """

    def run_a_from_sync(self, function, *args):
        """
        run a async fuction
        """

    def run_bg_task_advanced(self, task, *args, **kwargs):
        """
        proxi attr
        """

    def wait_for_bg_tasks(self, timeout=None):
        """
        proxi attr
        """

    def run_bg_task(self, task):
        """
                run a async fuction
                """
    def run_function(self, mod_function_name: Enum or tuple,
                     tb_run_function_with_state=True,
                     tb_run_with_specification='app',
                     args_=None,
                     kwargs_=None,
                     *args,
                     **kwargs) -> Result:

        """proxi attr"""

    async def a_run_function(self, mod_function_name: Enum or tuple,
                             tb_run_function_with_state=True,
                             tb_run_with_specification='app',
                             args_=None,
                             kwargs_=None,
                             *args,
                             **kwargs) -> Result:

        """proxi attr"""

    def fuction_runner(self, function, function_data: dict, args: list, kwargs: dict, t0=.0):
        """
        parameters = function_data.get('params')
        modular_name = function_data.get('module_name')
        function_name = function_data.get('func_name')
        mod_function_name = f"{modular_name}.{function_name}"

        proxi attr
        """

    async def a_fuction_runner(self, function, function_data: dict, args: list, kwargs: dict):
        """
        parameters = function_data.get('params')
        modular_name = function_data.get('module_name')
        function_name = function_data.get('func_name')
        mod_function_name = f"{modular_name}.{function_name}"

        proxi attr
        """

    async def run_http(self, mod_function_name: Enum or str or tuple, function_name=None, method="GET",
                       args_=None,
                       kwargs_=None,
                       *args, **kwargs):
        """run a function remote via http / https"""

    def run_any(self, mod_function_name: Enum or str or tuple, backwords_compability_variabel_string_holder=None,
                get_results=False, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None,
                kwargs_=None,
                *args, **kwargs):
        """proxi attr"""

    async def a_run_any(self, mod_function_name: Enum or str or tuple,
                        backwords_compability_variabel_string_holder=None,
                        get_results=False, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None,
                        kwargs_=None,
                        *args, **kwargs):
        """proxi attr"""

    def get_mod(self, name, spec='app') -> ModuleType or MainToolType:
        """proxi attr"""

    @staticmethod
    def print(text, *args, **kwargs):
        """proxi attr"""

    @staticmethod
    def sprint(text, *args, **kwargs):
        """proxi attr"""

    # ----------------------------------------------------------------
    # Decorators for the toolbox

    def _register_function(self, module_name, func_name, data):
        """proxi attr"""

    def _create_decorator(self, type_: str,
                          name: str = "",
                          mod_name: str = "",
                          level: int = -1,
                          restrict_in_virtual_mode: bool = False,
                          api: bool = False,
                          helper: str = "",
                          version: str or None = None,
                          initial=False,
                          exit_f=False,
                          test=True,
                          samples=None,
                          state=None,
                          pre_compute=None,
                          post_compute=None,
                          memory_cache=False,
                          file_cache=False,
                          row=False,
                          request_as_kwarg=False,
                          memory_cache_max_size=100,
                          memory_cache_ttl=300,
                          websocket_handler: str | None = None,):
        """proxi attr"""

        # data = {
        #     "type": type_,
        #     "module_name": module_name,
        #     "func_name": func_name,
        #     "level": level,
        #     "restrict_in_virtual_mode": restrict_in_virtual_mode,
        #     "func": func,
        #     "api": api,
        #     "helper": helper,
        #     "version": version,
        #     "initial": initial,
        #     "exit_f": exit_f,
        #     "__module__": func.__module__,
        #     "signature": sig,
        #     "params": params,
        #     "state": (
        #         False if len(params) == 0 else params[0] in ['self', 'state', 'app']) if state is None else state,
        #     "do_test": test,
        #     "samples": samples,
        #     "request_as_kwarg": request_as_kwarg,

    def tb(self, name=None,
           mod_name: str = "",
           helper: str = "",
           version: str or None = None,
           test: bool = True,
           restrict_in_virtual_mode: bool = False,
           api: bool = False,
           initial: bool = False,
           exit_f: bool = False,
           test_only: bool = False,
           memory_cache: bool = False,
           file_cache: bool = False,
           row=False,
           request_as_kwarg: bool = False,
           state: bool or None = None,
           level: int = 0,
           memory_cache_max_size: int = 100,
           memory_cache_ttl: int = 300,
           samples: list or dict or None = None,
           interface: ToolBoxInterfaces or None or str = None,
           pre_compute=None,
           post_compute=None,
           api_methods=None,
           websocket_handler: str | None = None,
           ):
        """
    A decorator for registering and configuring functions within a module.

    This decorator is used to wrap functions with additional functionality such as caching, API conversion, and lifecycle management (initialization and exit). It also handles the registration of the function in the module's function registry.

    Args:
        name (str, optional): The name to register the function under. Defaults to the function's own name.
        mod_name (str, optional): The name of the module the function belongs to.
        helper (str, optional): A helper string providing additional information about the function.
        version (str or None, optional): The version of the function or module.
        test (bool, optional): Flag to indicate if the function is for testing purposes.
        restrict_in_virtual_mode (bool, optional): Flag to restrict the function in virtual mode.
        api (bool, optional): Flag to indicate if the function is part of an API.
        initial (bool, optional): Flag to indicate if the function should be executed at initialization.
        exit_f (bool, optional): Flag to indicate if the function should be executed at exit.
        test_only (bool, optional): Flag to indicate if the function should only be used for testing.
        memory_cache (bool, optional): Flag to enable memory caching for the function.
        request_as_kwarg (bool, optional): Flag to get request if the fuction is calld from api.
        file_cache (bool, optional): Flag to enable file caching for the function.
        row (bool, optional): rather to auto wrap the result in Result type default False means no row data aka result type
        state (bool or None, optional): Flag to indicate if the function maintains state.
        level (int, optional): The level of the function, used for prioritization or categorization.
        memory_cache_max_size (int, optional): Maximum size of the memory cache.
        memory_cache_ttl (int, optional): Time-to-live for the memory cache entries.
        samples (list or dict or None, optional): Samples or examples of function usage.
        interface (str, optional): The interface type for the function.
        pre_compute (callable, optional): A function to be called before the main function.
        post_compute (callable, optional): A function to be called after the main function.
        api_methods (list[str], optional): default ["AUTO"] (GET if not params, POST if params) , GET, POST, PUT or DELETE.

    Returns:
        function: The decorated function with additional processing and registration capabilities.
    """
        if interface is None:
            interface = "tb"
        if test_only and 'test' not in self.id:
            return lambda *args, **kwargs: args
        return self._create_decorator(interface,
                                      name,
                                      mod_name,
                                      level=level,
                                      restrict_in_virtual_mode=restrict_in_virtual_mode,
                                      helper=helper,
                                      api=api,
                                      version=version,
                                      initial=initial,
                                      exit_f=exit_f,
                                      test=test,
                                      samples=samples,
                                      state=state,
                                      pre_compute=pre_compute,
                                      post_compute=post_compute,
                                      memory_cache=memory_cache,
                                      file_cache=file_cache,
                                      row=row,
                                      request_as_kwarg=request_as_kwarg,
                                      memory_cache_max_size=memory_cache_max_size,
                                      memory_cache_ttl=memory_cache_ttl)

    def print_functions(self, name=None):


        if not self.functions:
            return

        def helper(_functions):
            for func_name, data in _functions.items():
                if not isinstance(data, dict):
                    continue

                func_type = data.get('type', 'Unknown')
                func_level = 'r' if data['level'] == -1 else data['level']
                api_status = 'Api' if data.get('api', False) else 'Non-Api'

                print(f"  Function: {func_name}{data.get('signature', '()')}; "
                      f"Type: {func_type}, Level: {func_level}, {api_status}")

        if name is not None:
            functions = self.functions.get(name)
            if functions is not None:
                print(f"\nModule: {name}; Type: {functions.get('app_instance_type', 'Unknown')}")
                helper(functions)
                return
        for module, functions in self.functions.items():
            print(f"\nModule: {module}; Type: {functions.get('app_instance_type', 'Unknown')}")
            helper(functions)

    def save_autocompletion_dict(self):
        """proxi attr"""

    def get_autocompletion_dict(self):
        """proxi attr"""

    def get_username(self, get_input=False, default="loot") -> str:
        """proxi attr"""

    def save_registry_as_enums(self, directory: str, filename: str):
        """proxi attr"""

    async def execute_all_functions_(self, m_query='', f_query=''):
        print("Executing all functions")
        from ..extras import generate_test_cases
        all_data = {
            "modular_run": 0,
            "modular_fatal_error": 0,
            "errors": 0,
            "modular_sug": 0,
            "coverage": [],
            "total_coverage": {},
        }
        items = list(self.functions.items()).copy()
        for module_name, functions in items:
            infos = {
                "functions_run": 0,
                "functions_fatal_error": 0,
                "error": 0,
                "functions_sug": 0,
                'calls': {},
                'callse': {},
                "coverage": [0, 0],
            }
            all_data['modular_run'] += 1
            if not module_name.startswith(m_query):
                all_data['modular_sug'] += 1
                continue

            with Spinner(message=f"In {module_name}| "):
                f_items = list(functions.items()).copy()
                for function_name, function_data in f_items:
                    if not isinstance(function_data, dict):
                        continue
                    if not function_name.startswith(f_query):
                        continue
                    test: list = function_data.get('do_test')
                    # print(test, module_name, function_name, function_data)
                    infos["coverage"][0] += 1
                    if test is False:
                        continue

                    with Spinner(message=f"\t\t\t\t\t\tfuction {function_name}..."):
                        params: list = function_data.get('params')
                        sig: signature = function_data.get('signature')
                        state: bool = function_data.get('state')
                        samples: bool = function_data.get('samples')

                        test_kwargs_list = [{}]

                        if params is not None:
                            test_kwargs_list = samples if samples is not None else generate_test_cases(sig=sig)
                            # print(test_kwargs)
                            # print(test_kwargs[0])
                            # test_kwargs = test_kwargs_list[0]
                        # print(module_name, function_name, test_kwargs_list)
                        infos["coverage"][1] += 1
                        for test_kwargs in test_kwargs_list:
                            try:
                                # print(f"test Running {state=} |{module_name}.{function_name}")
                                result = await self.a_run_function((module_name, function_name),
                                                                   tb_run_function_with_state=state,
                                                                   **test_kwargs)
                                if not isinstance(result, Result):
                                    result = Result.ok(result)
                                if result.info.exec_code == 0:
                                    infos['calls'][function_name] = [test_kwargs, str(result)]
                                    infos['functions_sug'] += 1
                                else:
                                    infos['functions_sug'] += 1
                                    infos['error'] += 1
                                    infos['callse'][function_name] = [test_kwargs, str(result)]
                            except Exception as e:
                                infos['functions_fatal_error'] += 1
                                infos['callse'][function_name] = [test_kwargs, str(e)]
                            finally:
                                infos['functions_run'] += 1

                if infos['functions_run'] == infos['functions_sug']:
                    all_data['modular_sug'] += 1
                else:
                    all_data['modular_fatal_error'] += 1
                if infos['error'] > 0:
                    all_data['errors'] += infos['error']

                all_data[module_name] = infos
                if infos['coverage'][0] == 0:
                    c = 0
                else:
                    c = infos['coverage'][1] / infos['coverage'][0]
                all_data["coverage"].append(f"{module_name}:{c:.2f}\n")
        total_coverage = sum([float(t.split(":")[-1]) for t in all_data["coverage"]]) / len(all_data["coverage"])
        print(
            f"\n{all_data['modular_run']=}\n{all_data['modular_sug']=}\n{all_data['modular_fatal_error']=}\n{total_coverage=}")
        d = analyze_data(all_data)
        return Result.ok(data=all_data, data_info=d)

    async def execute_function_test(self, module_name: str, function_name: str,
                                    function_data: dict, test_kwargs: dict,
                                    profiler: cProfile.Profile) -> tuple[bool, str, dict, float]:
        start_time = time.time()
        with profile_section(profiler, hasattr(self, 'enable_profiling') and self.enable_profiling):
            try:
                result = await self.a_run_function(
                    (module_name, function_name),
                    tb_run_function_with_state=function_data.get('state'),
                    **test_kwargs
                )

                if not isinstance(result, Result):
                    result = Result.ok(result)

                success = result.info.exec_code == 0
                execution_time = time.time() - start_time
                return success, str(result), test_kwargs, execution_time
            except Exception as e:
                execution_time = time.time() - start_time
                return False, str(e), test_kwargs, execution_time

    async def process_function(self, module_name: str, function_name: str,
                               function_data: dict, profiler: cProfile.Profile) -> tuple[str, ModuleInfo]:
        start_time = time.time()
        info = ModuleInfo()

        with profile_section(profiler, hasattr(self, 'enable_profiling') and self.enable_profiling):
            if not isinstance(function_data, dict):
                return function_name, info

            test = function_data.get('do_test')
            info.coverage[0] += 1

            if test is False:
                return function_name, info

            params = function_data.get('params')
            sig = function_data.get('signature')
            samples = function_data.get('samples')

            test_kwargs_list = [{}] if params is None else (
                samples if samples is not None else generate_test_cases(sig=sig)
            )

            info.coverage[1] += 1

            # Create tasks for all test cases
            tasks = [
                self.execute_function_test(module_name, function_name, function_data, test_kwargs, profiler)
                for test_kwargs in test_kwargs_list
            ]

            # Execute all tests concurrently
            results = await asyncio.gather(*tasks)

            total_execution_time = 0
            for success, result_str, test_kwargs, execution_time in results:
                info.functions_run += 1
                total_execution_time += execution_time

                if success:
                    info.functions_sug += 1
                    info.calls[function_name] = [test_kwargs, result_str]
                else:
                    info.functions_sug += 1
                    info.error += 1
                    info.callse[function_name] = [test_kwargs, result_str]

            info.execution_time = time.time() - start_time
            return function_name, info

    async def process_module(self, module_name: str, functions: dict,
                             f_query: str, profiler: cProfile.Profile) -> tuple[str, ModuleInfo]:
        start_time = time.time()

        with profile_section(profiler, hasattr(self, 'enable_profiling') and self.enable_profiling):
            async with asyncio.Semaphore(mp.cpu_count()):
                tasks = [
                    self.process_function(module_name, fname, fdata, profiler)
                    for fname, fdata in functions.items()
                    if fname.startswith(f_query)
                ]

                if not tasks:
                    return module_name, ModuleInfo()

                results = await asyncio.gather(*tasks)

                # Combine results from all functions in the module
                combined_info = ModuleInfo()
                total_execution_time = 0

                for _, info in results:
                    combined_info.functions_run += info.functions_run
                    combined_info.functions_fatal_error += info.functions_fatal_error
                    combined_info.error += info.error
                    combined_info.functions_sug += info.functions_sug
                    combined_info.calls.update(info.calls)
                    combined_info.callse.update(info.callse)
                    combined_info.coverage[0] += info.coverage[0]
                    combined_info.coverage[1] += info.coverage[1]
                    total_execution_time += info.execution_time

                combined_info.execution_time = time.time() - start_time
                return module_name, combined_info

    async def execute_all_functions(self, m_query='', f_query='', enable_profiling=True):
        """
        Execute all functions with parallel processing and optional profiling.

        Args:
            m_query (str): Module name query filter
            f_query (str): Function name query filter
            enable_profiling (bool): Enable detailed profiling information
        """
        print("Executing all functions in parallel" + (" with profiling" if enable_profiling else ""))

        start_time = time.time()
        stats = ExecutionStats()
        items = list(self.functions.items()).copy()

        # Set up profiling
        self.enable_profiling = enable_profiling
        profiler = cProfile.Profile()

        with profile_section(profiler, enable_profiling):
            # Filter modules based on query
            filtered_modules = [
                (mname, mfuncs) for mname, mfuncs in items
                if mname.startswith(m_query)
            ]

            stats.modular_run = len(filtered_modules)

            # Process all modules concurrently
            async with asyncio.Semaphore(mp.cpu_count()):
                tasks = [
                    self.process_module(mname, mfuncs, f_query, profiler)
                    for mname, mfuncs in filtered_modules
                ]

                results = await asyncio.gather(*tasks)

            # Combine results and calculate statistics
            for module_name, info in results:
                if info.functions_run == info.functions_sug:
                    stats.modular_sug += 1
                else:
                    stats.modular_fatal_error += 1

                stats.errors += info.error

                # Calculate coverage
                coverage = (info.coverage[1] / info.coverage[0]) if info.coverage[0] > 0 else 0
                stats.coverage.append(f"{module_name}:{coverage:.2f}\n")

                # Store module info
                stats.__dict__[module_name] = info

            # Calculate total coverage
            total_coverage = (
                sum(float(t.split(":")[-1]) for t in stats.coverage) / len(stats.coverage)
                if stats.coverage else 0
            )

            stats.total_execution_time = time.time() - start_time

            # Generate profiling stats if enabled
            if enable_profiling:
                s = io.StringIO()
                ps = pstats.Stats(profiler, stream=s).sort_stats('cumulative')
                ps.print_stats()
                stats.profiling_data = {
                    'detailed_stats': s.getvalue(),
                    'total_time': stats.total_execution_time,
                    'function_count': stats.modular_run,
                    'successful_functions': stats.modular_sug
                }

            print(
                f"\n{stats.modular_run=}"
                f"\n{stats.modular_sug=}"
                f"\n{stats.modular_fatal_error=}"
                f"\n{total_coverage=}"
                f"\nTotal execution time: {stats.total_execution_time:.2f}s"
            )

            if enable_profiling:
                print("\nProfiling Summary:")
                print(f"{'=' * 50}")
                print("Top 10 time-consuming functions:")
                ps.print_stats(10)

            analyzed_data = analyze_data(stats.__dict__)
            return Result.ok(data=stats.__dict__, data_info=analyzed_data)

debug property writable

proxi attr

a_exit() async

proxi attr

Source code in toolboxv2/utils/system/types.py

async def a_exit(self):
    """proxi attr"""

a_fuction_runner(function, function_data, args, kwargs) async

parameters = function_data.get('params') modular_name = function_data.get('module_name') function_name = function_data.get('func_name') mod_function_name = f"{modular_name}.{function_name}"

proxi attr

Source code in toolboxv2/utils/system/types.py

async def a_fuction_runner(self, function, function_data: dict, args: list, kwargs: dict):
    """
    parameters = function_data.get('params')
    modular_name = function_data.get('module_name')
    function_name = function_data.get('func_name')
    mod_function_name = f"{modular_name}.{function_name}"

    proxi attr
    """

a_remove_mod(mod_name, spec='app', delete=True) async

proxi attr

Source code in toolboxv2/utils/system/types.py

async def a_remove_mod(self, mod_name, spec='app', delete=True):
    """proxi attr"""

a_run_any(mod_function_name, backwords_compability_variabel_string_holder=None, get_results=False, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None, kwargs_=None, *args, **kwargs) async

proxi attr

Source code in toolboxv2/utils/system/types.py

async def a_run_any(self, mod_function_name: Enum or str or tuple,
                    backwords_compability_variabel_string_holder=None,
                    get_results=False, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None,
                    kwargs_=None,
                    *args, **kwargs):
    """proxi attr"""

a_run_function(mod_function_name, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None, kwargs_=None, *args, **kwargs) async

proxi attr

Source code in toolboxv2/utils/system/types.py

async def a_run_function(self, mod_function_name: Enum or tuple,
                         tb_run_function_with_state=True,
                         tb_run_with_specification='app',
                         args_=None,
                         kwargs_=None,
                         *args,
                         **kwargs) -> Result:

    """proxi attr"""

debug_rains(e)

proxi attr

Source code in toolboxv2/utils/system/types.py

def debug_rains(self, e):
    """proxi attr"""

disconnect(*args, **kwargs) async staticmethod

proxi attr

Source code in toolboxv2/utils/system/types.py

@staticmethod
async def disconnect(*args, **kwargs):
    """proxi attr"""

execute_all_functions(m_query='', f_query='', enable_profiling=True) async

Execute all functions with parallel processing and optional profiling.

Parameters:

Name	Type	Description	Default
m_query	str	Module name query filter	''
f_query	str	Function name query filter	''
enable_profiling	bool	Enable detailed profiling information	True

Source code in toolboxv2/utils/system/types.py

async def execute_all_functions(self, m_query='', f_query='', enable_profiling=True):
    """
    Execute all functions with parallel processing and optional profiling.

    Args:
        m_query (str): Module name query filter
        f_query (str): Function name query filter
        enable_profiling (bool): Enable detailed profiling information
    """
    print("Executing all functions in parallel" + (" with profiling" if enable_profiling else ""))

    start_time = time.time()
    stats = ExecutionStats()
    items = list(self.functions.items()).copy()

    # Set up profiling
    self.enable_profiling = enable_profiling
    profiler = cProfile.Profile()

    with profile_section(profiler, enable_profiling):
        # Filter modules based on query
        filtered_modules = [
            (mname, mfuncs) for mname, mfuncs in items
            if mname.startswith(m_query)
        ]

        stats.modular_run = len(filtered_modules)

        # Process all modules concurrently
        async with asyncio.Semaphore(mp.cpu_count()):
            tasks = [
                self.process_module(mname, mfuncs, f_query, profiler)
                for mname, mfuncs in filtered_modules
            ]

            results = await asyncio.gather(*tasks)

        # Combine results and calculate statistics
        for module_name, info in results:
            if info.functions_run == info.functions_sug:
                stats.modular_sug += 1
            else:
                stats.modular_fatal_error += 1

            stats.errors += info.error

            # Calculate coverage
            coverage = (info.coverage[1] / info.coverage[0]) if info.coverage[0] > 0 else 0
            stats.coverage.append(f"{module_name}:{coverage:.2f}\n")

            # Store module info
            stats.__dict__[module_name] = info

        # Calculate total coverage
        total_coverage = (
            sum(float(t.split(":")[-1]) for t in stats.coverage) / len(stats.coverage)
            if stats.coverage else 0
        )

        stats.total_execution_time = time.time() - start_time

        # Generate profiling stats if enabled
        if enable_profiling:
            s = io.StringIO()
            ps = pstats.Stats(profiler, stream=s).sort_stats('cumulative')
            ps.print_stats()
            stats.profiling_data = {
                'detailed_stats': s.getvalue(),
                'total_time': stats.total_execution_time,
                'function_count': stats.modular_run,
                'successful_functions': stats.modular_sug
            }

        print(
            f"\n{stats.modular_run=}"
            f"\n{stats.modular_sug=}"
            f"\n{stats.modular_fatal_error=}"
            f"\n{total_coverage=}"
            f"\nTotal execution time: {stats.total_execution_time:.2f}s"
        )

        if enable_profiling:
            print("\nProfiling Summary:")
            print(f"{'=' * 50}")
            print("Top 10 time-consuming functions:")
            ps.print_stats(10)

        analyzed_data = analyze_data(stats.__dict__)
        return Result.ok(data=stats.__dict__, data_info=analyzed_data)

exit()

proxi attr

Source code in toolboxv2/utils/system/types.py

def exit(self):
    """proxi attr"""

exit_main(*args, **kwargs) staticmethod

proxi attr

Source code in toolboxv2/utils/system/types.py

@staticmethod
def exit_main(*args, **kwargs):
    """proxi attr"""

footprint(update_tracking=True)

Erfasst den aktuellen Ressourcen-Footprint der Toolbox-Instanz.

Parameters:

Name	Type	Description	Default
update_tracking	bool	Wenn True, aktualisiert Min/Max/Avg-Tracking	True

Returns:

Type	Description
FootprintMetrics	FootprintMetrics mit allen erfassten Metriken

Source code in toolboxv2/utils/system/types.py

def footprint(self, update_tracking: bool = True) -> FootprintMetrics:
    """
    Erfasst den aktuellen Ressourcen-Footprint der Toolbox-Instanz.

    Args:
        update_tracking: Wenn True, aktualisiert Min/Max/Avg-Tracking

    Returns:
        FootprintMetrics mit allen erfassten Metriken
    """
    current_time = time.time()
    uptime_seconds = current_time - self._footprint_start_time

    # Formatierte Uptime
    uptime_delta = timedelta(seconds=int(uptime_seconds))
    uptime_formatted = str(uptime_delta)

    # Memory Metrics (in MB)
    try:
        mem_info = self._process.memory_info()
        memory_current = mem_info.rss / (1024 * 1024)  # Bytes zu MB
        memory_percent = self._process.memory_percent()

        if update_tracking:
            self._update_metric_tracking('memory', memory_current)

        memory_max = self._footprint_metrics['memory']['max']
        memory_min = self._footprint_metrics['memory']['min']
        if memory_min == float('inf'):
            memory_min = memory_current
    except (psutil.NoSuchProcess, psutil.AccessDenied):
        memory_current = memory_max = memory_min = memory_percent = 0

    # CPU Metrics
    try:
        cpu_percent_current = self._process.cpu_percent(interval=0.1)
        cpu_times = self._process.cpu_times()
        cpu_time_seconds = cpu_times.user + cpu_times.system

        if update_tracking:
            self._update_metric_tracking('cpu', cpu_percent_current)

        cpu_percent_max = self._footprint_metrics['cpu']['max']
        cpu_percent_min = self._footprint_metrics['cpu']['min']
        cpu_percent_avg = self._get_metric_avg('cpu')

        if cpu_percent_min == float('inf'):
            cpu_percent_min = cpu_percent_current
    except (psutil.NoSuchProcess, psutil.AccessDenied):
        cpu_percent_current = cpu_percent_max = 0
        cpu_percent_min = cpu_percent_avg = cpu_time_seconds = 0

    # Disk I/O Metrics (in MB)
    try:
        io_counters = self._process.io_counters()
        disk_read_bytes = io_counters.read_bytes - self._initial_disk_read
        disk_write_bytes = io_counters.write_bytes - self._initial_disk_write

        disk_read_mb = disk_read_bytes / (1024 * 1024)
        disk_write_mb = disk_write_bytes / (1024 * 1024)

        if update_tracking:
            self._update_metric_tracking('disk_read', disk_read_mb)
            self._update_metric_tracking('disk_write', disk_write_mb)

        disk_read_max = self._footprint_metrics['disk_read']['max']
        disk_read_min = self._footprint_metrics['disk_read']['min']
        disk_write_max = self._footprint_metrics['disk_write']['max']
        disk_write_min = self._footprint_metrics['disk_write']['min']

        if disk_read_min == float('inf'):
            disk_read_min = disk_read_mb
        if disk_write_min == float('inf'):
            disk_write_min = disk_write_mb
    except (AttributeError, OSError, psutil.NoSuchProcess, psutil.AccessDenied):
        disk_read_mb = disk_write_mb = 0
        disk_read_max = disk_read_min = disk_write_max = disk_write_min = 0

    # Network I/O Metrics (in MB)
    try:
        net_io = psutil.net_io_counters()
        network_sent_bytes = net_io.bytes_sent - self._initial_network_sent
        network_recv_bytes = net_io.bytes_recv - self._initial_network_recv

        network_sent_mb = network_sent_bytes / (1024 * 1024)
        network_recv_mb = network_recv_bytes / (1024 * 1024)

        if update_tracking:
            self._update_metric_tracking('network_sent', network_sent_mb)
            self._update_metric_tracking('network_recv', network_recv_mb)

        network_sent_max = self._footprint_metrics['network_sent']['max']
        network_sent_min = self._footprint_metrics['network_sent']['min']
        network_recv_max = self._footprint_metrics['network_recv']['max']
        network_recv_min = self._footprint_metrics['network_recv']['min']

        if network_sent_min == float('inf'):
            network_sent_min = network_sent_mb
        if network_recv_min == float('inf'):
            network_recv_min = network_recv_mb
    except (AttributeError, OSError):
        network_sent_mb = network_recv_mb = 0
        network_sent_max = network_sent_min = 0
        network_recv_max = network_recv_min = 0

    # Process Info
    try:
        process_id = self._process.pid
        threads = self._process.num_threads()
        open_files_path = [str(x.path).replace("\\", "/") for x in self._process.open_files()]
        connections_uri = [f"{x.laddr}:{x.raddr} {str(x.status)}" for x in self._process.connections()]

        open_files = len(open_files_path)
        connections = len(connections_uri)
    except (psutil.NoSuchProcess, psutil.AccessDenied):
        process_id = os.getpid()
        threads = open_files = connections = 0
        open_files_path = []
        connections_uri = []

    return FootprintMetrics(
        start_time=self._footprint_start_time,
        uptime_seconds=uptime_seconds,
        uptime_formatted=uptime_formatted,
        memory_current=memory_current,
        memory_max=memory_max,
        memory_min=memory_min,
        memory_percent=memory_percent,
        cpu_percent_current=cpu_percent_current,
        cpu_percent_max=cpu_percent_max,
        cpu_percent_min=cpu_percent_min,
        cpu_percent_avg=cpu_percent_avg,
        cpu_time_seconds=cpu_time_seconds,
        disk_read_mb=disk_read_mb,
        disk_write_mb=disk_write_mb,
        disk_read_max=disk_read_max,
        disk_read_min=disk_read_min,
        disk_write_max=disk_write_max,
        disk_write_min=disk_write_min,
        network_sent_mb=network_sent_mb,
        network_recv_mb=network_recv_mb,
        network_sent_max=network_sent_max,
        network_sent_min=network_sent_min,
        network_recv_max=network_recv_max,
        network_recv_min=network_recv_min,
        process_id=process_id,
        threads=threads,
        open_files=open_files,
        connections=connections,
        open_files_path=open_files_path,
        connections_uri=connections_uri,
    )

fuction_runner(function, function_data, args, kwargs, t0=0.0)

parameters = function_data.get('params') modular_name = function_data.get('module_name') function_name = function_data.get('func_name') mod_function_name = f"{modular_name}.{function_name}"

proxi attr

Source code in toolboxv2/utils/system/types.py

def fuction_runner(self, function, function_data: dict, args: list, kwargs: dict, t0=.0):
    """
    parameters = function_data.get('params')
    modular_name = function_data.get('module_name')
    function_name = function_data.get('func_name')
    mod_function_name = f"{modular_name}.{function_name}"

    proxi attr
    """

get_all_mods(working_dir='mods', path_to='./runtime')

proxi attr

Source code in toolboxv2/utils/system/types.py

def get_all_mods(self, working_dir="mods", path_to="./runtime"):
    """proxi attr"""

get_autocompletion_dict()

proxi attr

Source code in toolboxv2/utils/system/types.py

def get_autocompletion_dict(self):
    """proxi attr"""

get_function(name, **kwargs)

Kwargs for _get_function metadata:: return the registered function dictionary stateless: (function_data, None), 0 stateful: (function_data, higher_order_function), 0 state::boolean specification::str default app

Source code in toolboxv2/utils/system/types.py

def get_function(self, name: Enum or tuple, **kwargs):
    """
    Kwargs for _get_function
        metadata:: return the registered function dictionary
            stateless: (function_data, None), 0
            stateful: (function_data, higher_order_function), 0
        state::boolean
            specification::str default app
    """

get_mod(name, spec='app')

proxi attr

Source code in toolboxv2/utils/system/types.py

def get_mod(self, name, spec='app') -> ModuleType or MainToolType:
    """proxi attr"""

get_username(get_input=False, default='loot')

proxi attr

Source code in toolboxv2/utils/system/types.py

def get_username(self, get_input=False, default="loot") -> str:
    """proxi attr"""

hide_console(*args, **kwargs) async staticmethod

proxi attr

Source code in toolboxv2/utils/system/types.py

@staticmethod
async def hide_console(*args, **kwargs):
    """proxi attr"""

inplace_load_instance(mod_name, loc='toolboxv2.mods.', spec='app', save=True)

proxi attr

Source code in toolboxv2/utils/system/types.py

def inplace_load_instance(self, mod_name, loc="toolboxv2.mods.", spec='app', save=True):
    """proxi attr"""

load_all_mods_in_file(working_dir='mods') async

proxi attr

Source code in toolboxv2/utils/system/types.py

async def load_all_mods_in_file(self, working_dir="mods"):
    """proxi attr"""

load_external_mods() async

proxi attr

Source code in toolboxv2/utils/system/types.py

async def load_external_mods(self):
    """proxi attr"""

load_mod(mod_name, mlm='I', **kwargs)

proxi attr

Source code in toolboxv2/utils/system/types.py

def load_mod(self, mod_name: str, mlm='I', **kwargs):
    """proxi attr"""

mod_online(mod_name, installed=False)

proxi attr

Source code in toolboxv2/utils/system/types.py

def mod_online(self, mod_name, installed=False):
    """proxi attr"""

print(text, *args, **kwargs) staticmethod

proxi attr

Source code in toolboxv2/utils/system/types.py

@staticmethod
def print(text, *args, **kwargs):
    """proxi attr"""

print_footprint(detailed=True)

Gibt den Footprint formatiert aus.

Parameters:

Name	Type	Description	Default
detailed	bool	Wenn True, zeigt alle Details, sonst nur Zusammenfassung	True

Returns:

Type	Description
str	Formatierter Footprint-String

Source code in toolboxv2/utils/system/types.py

def print_footprint(self, detailed: bool = True) -> str:
    """
    Gibt den Footprint formatiert aus.

    Args:
        detailed: Wenn True, zeigt alle Details, sonst nur Zusammenfassung

    Returns:
        Formatierter Footprint-String
    """
    metrics = self.footprint()

    output = [
        "=" * 70,
        f"TOOLBOX FOOTPRINT - {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}",
        "=" * 70,
        f"\n📊 UPTIME",
        f"  Runtime: {metrics.uptime_formatted}",
        f"  Seconds: {metrics.uptime_seconds:.2f}s",
        f"\n💾 MEMORY USAGE",
        f"  Current:  {metrics.memory_current:.2f} MB ({metrics.memory_percent:.2f}%)",
        f"  Maximum:  {metrics.memory_max:.2f} MB",
        f"  Minimum:  {metrics.memory_min:.2f} MB",
    ]

    if detailed:
        helper_ = '\n\t- '.join(metrics.open_files_path)
        helper__ = '\n\t- '.join(metrics.connections_uri)
        output.extend([
            f"\n⚙️  CPU USAGE",
            f"  Current:  {metrics.cpu_percent_current:.2f}%",
            f"  Maximum:  {metrics.cpu_percent_max:.2f}%",
            f"  Minimum:  {metrics.cpu_percent_min:.2f}%",
            f"  Average:  {metrics.cpu_percent_avg:.2f}%",
            f"  CPU Time: {metrics.cpu_time_seconds:.2f}s",
            f"\n💿 DISK I/O",
            f"  Read:     {metrics.disk_read_mb:.2f} MB (Max: {metrics.disk_read_max:.2f}, Min: {metrics.disk_read_min:.2f})",
            f"  Write:    {metrics.disk_write_mb:.2f} MB (Max: {metrics.disk_write_max:.2f}, Min: {metrics.disk_write_min:.2f})",
            f"\n🌐 NETWORK I/O",
            f"  Sent:     {metrics.network_sent_mb:.2f} MB (Max: {metrics.network_sent_max:.2f}, Min: {metrics.network_sent_min:.2f})",
            f"  Received: {metrics.network_recv_mb:.2f} MB (Max: {metrics.network_recv_max:.2f}, Min: {metrics.network_recv_min:.2f})",
            f"\n🔧 PROCESS INFO",
            f"  PID:         {metrics.process_id}",
            f"  Threads:     {metrics.threads}",
            f"\n📂 OPEN FILES",
            f"  Open Files:  {metrics.open_files}",
            f"  Open Files Path: \n\t- {helper_}",
            f"\n🔗 NETWORK CONNECTIONS",
            f"  Connections: {metrics.connections}",
            f"  Connections URI: \n\t- {helper__}",
        ])

    output.append("=" * 70)

    return "\n".join(output)

print_ok()

proxi attr

Source code in toolboxv2/utils/system/types.py

def print_ok(self):
    """proxi attr"""
    self.logger.info("OK")

reload_mod(mod_name, spec='app', is_file=True, loc='toolboxv2.mods.')

proxi attr

Source code in toolboxv2/utils/system/types.py

def reload_mod(self, mod_name, spec='app', is_file=True, loc="toolboxv2.mods."):
    """proxi attr"""

remove_mod(mod_name, spec='app', delete=True)

proxi attr

Source code in toolboxv2/utils/system/types.py

def remove_mod(self, mod_name, spec='app', delete=True):
    """proxi attr"""

rrun_flows(name, **kwargs)

proxi attr

Source code in toolboxv2/utils/system/types.py

def rrun_flows(self, name, **kwargs):
    """proxi attr"""

run_a_from_sync(function, *args)

run a async fuction

Source code in toolboxv2/utils/system/types.py

def run_a_from_sync(self, function, *args):
    """
    run a async fuction
    """

run_any(mod_function_name, backwords_compability_variabel_string_holder=None, get_results=False, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None, kwargs_=None, *args, **kwargs)

proxi attr

Source code in toolboxv2/utils/system/types.py

def run_any(self, mod_function_name: Enum or str or tuple, backwords_compability_variabel_string_holder=None,
            get_results=False, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None,
            kwargs_=None,
            *args, **kwargs):
    """proxi attr"""

run_bg_task(task)

run a async fuction

Source code in toolboxv2/utils/system/types.py

def run_bg_task(self, task):
    """
            run a async fuction
            """

run_bg_task_advanced(task, *args, **kwargs)

proxi attr

Source code in toolboxv2/utils/system/types.py

def run_bg_task_advanced(self, task, *args, **kwargs):
    """
    proxi attr
    """

run_flows(name, **kwargs) async

proxi attr

Source code in toolboxv2/utils/system/types.py

async def run_flows(self, name, **kwargs):
    """proxi attr"""

run_function(mod_function_name, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None, kwargs_=None, *args, **kwargs)

proxi attr

Source code in toolboxv2/utils/system/types.py

def run_function(self, mod_function_name: Enum or tuple,
                 tb_run_function_with_state=True,
                 tb_run_with_specification='app',
                 args_=None,
                 kwargs_=None,
                 *args,
                 **kwargs) -> Result:

    """proxi attr"""

run_http(mod_function_name, function_name=None, method='GET', args_=None, kwargs_=None, *args, **kwargs) async

run a function remote via http / https

Source code in toolboxv2/utils/system/types.py

async def run_http(self, mod_function_name: Enum or str or tuple, function_name=None, method="GET",
                   args_=None,
                   kwargs_=None,
                   *args, **kwargs):
    """run a function remote via http / https"""

save_autocompletion_dict()

proxi attr

Source code in toolboxv2/utils/system/types.py

def save_autocompletion_dict(self):
    """proxi attr"""

save_exit()

proxi attr

Source code in toolboxv2/utils/system/types.py

def save_exit(self):
    """proxi attr"""

save_initialized_module(tools_class, spec)

proxi attr

Source code in toolboxv2/utils/system/types.py

def save_initialized_module(self, tools_class, spec):
    """proxi attr"""

save_instance(instance, modular_id, spec='app', instance_type='file/application', tools_class=None)

proxi attr

Source code in toolboxv2/utils/system/types.py

def save_instance(self, instance, modular_id, spec='app', instance_type="file/application", tools_class=None):
    """proxi attr"""

save_load(modname, spec='app')

proxi attr

Source code in toolboxv2/utils/system/types.py

def save_load(self, modname, spec='app'):
    """proxi attr"""

save_registry_as_enums(directory, filename)

proxi attr

Source code in toolboxv2/utils/system/types.py

def save_registry_as_enums(self, directory: str, filename: str):
    """proxi attr"""

set_flows(r)

proxi attr

Source code in toolboxv2/utils/system/types.py

def set_flows(self, r):
    """proxi attr"""

set_logger(debug=False)

proxi attr

Source code in toolboxv2/utils/system/types.py

def set_logger(self, debug=False):
    """proxi attr"""

show_console(*args, **kwargs) async staticmethod

proxi attr

Source code in toolboxv2/utils/system/types.py

@staticmethod
async def show_console(*args, **kwargs):
    """proxi attr"""

sprint(text, *args, **kwargs) staticmethod

proxi attr

Source code in toolboxv2/utils/system/types.py

@staticmethod
def sprint(text, *args, **kwargs):
    """proxi attr"""

tb(name=None, mod_name='', helper='', version=None, test=True, restrict_in_virtual_mode=False, api=False, initial=False, exit_f=False, test_only=False, memory_cache=False, file_cache=False, row=False, request_as_kwarg=False, state=None, level=0, memory_cache_max_size=100, memory_cache_ttl=300, samples=None, interface=None, pre_compute=None, post_compute=None, api_methods=None, websocket_handler=None)

A decorator for registering and configuring functions within a module.

This decorator is used to wrap functions with additional functionality such as caching, API conversion, and lifecycle management (initialization and exit). It also handles the registration of the function in the module's function registry.

Parameters:

Name	Type	Description	Default
name	str	The name to register the function under. Defaults to the function's own name.	None
mod_name	str	The name of the module the function belongs to.	''
helper	str	A helper string providing additional information about the function.	''
version	str or None	The version of the function or module.	None
test	bool	Flag to indicate if the function is for testing purposes.	True
restrict_in_virtual_mode	bool	Flag to restrict the function in virtual mode.	False
api	bool	Flag to indicate if the function is part of an API.	False
initial	bool	Flag to indicate if the function should be executed at initialization.	False
exit_f	bool	Flag to indicate if the function should be executed at exit.	False
test_only	bool	Flag to indicate if the function should only be used for testing.	False
memory_cache	bool	Flag to enable memory caching for the function.	False
request_as_kwarg	bool	Flag to get request if the fuction is calld from api.	False
file_cache	bool	Flag to enable file caching for the function.	False
row	bool	rather to auto wrap the result in Result type default False means no row data aka result type	False
state	bool or None	Flag to indicate if the function maintains state.	None
level	int	The level of the function, used for prioritization or categorization.	0
memory_cache_max_size	int	Maximum size of the memory cache.	100
memory_cache_ttl	int	Time-to-live for the memory cache entries.	300
samples	list or dict or None	Samples or examples of function usage.	None
interface	str	The interface type for the function.	None
pre_compute	callable	A function to be called before the main function.	None
post_compute	callable	A function to be called after the main function.	None
api_methods	list[str]	default ["AUTO"] (GET if not params, POST if params) , GET, POST, PUT or DELETE.	None

Returns:

Name	Type	Description
function		The decorated function with additional processing and registration capabilities.

Source code in toolboxv2/utils/system/types.py

def tb(self, name=None,
       mod_name: str = "",
       helper: str = "",
       version: str or None = None,
       test: bool = True,
       restrict_in_virtual_mode: bool = False,
       api: bool = False,
       initial: bool = False,
       exit_f: bool = False,
       test_only: bool = False,
       memory_cache: bool = False,
       file_cache: bool = False,
       row=False,
       request_as_kwarg: bool = False,
       state: bool or None = None,
       level: int = 0,
       memory_cache_max_size: int = 100,
       memory_cache_ttl: int = 300,
       samples: list or dict or None = None,
       interface: ToolBoxInterfaces or None or str = None,
       pre_compute=None,
       post_compute=None,
       api_methods=None,
       websocket_handler: str | None = None,
       ):
    """
A decorator for registering and configuring functions within a module.

This decorator is used to wrap functions with additional functionality such as caching, API conversion, and lifecycle management (initialization and exit). It also handles the registration of the function in the module's function registry.

Args:
    name (str, optional): The name to register the function under. Defaults to the function's own name.
    mod_name (str, optional): The name of the module the function belongs to.
    helper (str, optional): A helper string providing additional information about the function.
    version (str or None, optional): The version of the function or module.
    test (bool, optional): Flag to indicate if the function is for testing purposes.
    restrict_in_virtual_mode (bool, optional): Flag to restrict the function in virtual mode.
    api (bool, optional): Flag to indicate if the function is part of an API.
    initial (bool, optional): Flag to indicate if the function should be executed at initialization.
    exit_f (bool, optional): Flag to indicate if the function should be executed at exit.
    test_only (bool, optional): Flag to indicate if the function should only be used for testing.
    memory_cache (bool, optional): Flag to enable memory caching for the function.
    request_as_kwarg (bool, optional): Flag to get request if the fuction is calld from api.
    file_cache (bool, optional): Flag to enable file caching for the function.
    row (bool, optional): rather to auto wrap the result in Result type default False means no row data aka result type
    state (bool or None, optional): Flag to indicate if the function maintains state.
    level (int, optional): The level of the function, used for prioritization or categorization.
    memory_cache_max_size (int, optional): Maximum size of the memory cache.
    memory_cache_ttl (int, optional): Time-to-live for the memory cache entries.
    samples (list or dict or None, optional): Samples or examples of function usage.
    interface (str, optional): The interface type for the function.
    pre_compute (callable, optional): A function to be called before the main function.
    post_compute (callable, optional): A function to be called after the main function.
    api_methods (list[str], optional): default ["AUTO"] (GET if not params, POST if params) , GET, POST, PUT or DELETE.

Returns:
    function: The decorated function with additional processing and registration capabilities.
"""
    if interface is None:
        interface = "tb"
    if test_only and 'test' not in self.id:
        return lambda *args, **kwargs: args
    return self._create_decorator(interface,
                                  name,
                                  mod_name,
                                  level=level,
                                  restrict_in_virtual_mode=restrict_in_virtual_mode,
                                  helper=helper,
                                  api=api,
                                  version=version,
                                  initial=initial,
                                  exit_f=exit_f,
                                  test=test,
                                  samples=samples,
                                  state=state,
                                  pre_compute=pre_compute,
                                  post_compute=post_compute,
                                  memory_cache=memory_cache,
                                  file_cache=file_cache,
                                  row=row,
                                  request_as_kwarg=request_as_kwarg,
                                  memory_cache_max_size=memory_cache_max_size,
                                  memory_cache_ttl=memory_cache_ttl)

wait_for_bg_tasks(timeout=None)

proxi attr

Source code in toolboxv2/utils/system/types.py

def wait_for_bg_tasks(self, timeout=None):
    """
    proxi attr
    """

watch_mod(mod_name, spec='app', loc='toolboxv2.mods.', use_thread=True, path_name=None)

proxi attr

Source code in toolboxv2/utils/system/types.py

def watch_mod(self, mod_name, spec='app', loc="toolboxv2.mods.", use_thread=True, path_name=None):
    """proxi attr"""

web_context()

returns the build index ( toolbox web component )

Source code in toolboxv2/utils/system/types.py

def web_context(self) -> str:
    """returns the build index ( toolbox web component )"""

FootprintMetrics

Dataclass für Footprint-Metriken

Source code in toolboxv2/utils/system/types.py

@dataclass
class FootprintMetrics:
    """Dataclass für Footprint-Metriken"""
    # Uptime
    start_time: float
    uptime_seconds: float
    uptime_formatted: str

    # Memory (in MB)
    memory_current: float
    memory_max: float
    memory_min: float
    memory_percent: float

    # CPU
    cpu_percent_current: float
    cpu_percent_max: float
    cpu_percent_min: float
    cpu_percent_avg: float
    cpu_time_seconds: float

    # Disk I/O (in MB)
    disk_read_mb: float
    disk_write_mb: float
    disk_read_max: float
    disk_read_min: float
    disk_write_max: float
    disk_write_min: float

    # Network I/O (in MB)
    network_sent_mb: float
    network_recv_mb: float
    network_sent_max: float
    network_sent_min: float
    network_recv_max: float
    network_recv_min: float

    # Additional Info
    process_id: int
    threads: int
    open_files: int
    connections: int

    open_files_path: list[str]
    connections_uri: list[str]

    def to_dict(self) -> Dict[str, Any]:
        """Konvertiert Metriken in Dictionary"""
        return {
            'uptime': {
                'seconds': self.uptime_seconds,
                'formatted': self.uptime_formatted,
            },
            'memory': {
                'current_mb': self.memory_current,
                'max_mb': self.memory_max,
                'min_mb': self.memory_min,
                'percent': self.memory_percent,
            },
            'cpu': {
                'current_percent': self.cpu_percent_current,
                'max_percent': self.cpu_percent_max,
                'min_percent': self.cpu_percent_min,
                'avg_percent': self.cpu_percent_avg,
                'time_seconds': self.cpu_time_seconds,
            },
            'disk': {
                'read_mb': self.disk_read_mb,
                'write_mb': self.disk_write_mb,
                'read_max_mb': self.disk_read_max,
                'read_min_mb': self.disk_read_min,
                'write_max_mb': self.disk_write_max,
                'write_min_mb': self.disk_write_min,
            },
            'network': {
                'sent_mb': self.network_sent_mb,
                'recv_mb': self.network_recv_mb,
                'sent_max_mb': self.network_sent_max,
                'sent_min_mb': self.network_sent_min,
                'recv_max_mb': self.network_recv_max,
                'recv_min_mb': self.network_recv_min,
            },
            'process': {
                'pid': self.process_id,
                'threads': self.threads,
                'open_files': self.open_files,
                'connections': self.connections,
            }
        }

to_dict()

Konvertiert Metriken in Dictionary

Source code in toolboxv2/utils/system/types.py

def to_dict(self) -> Dict[str, Any]:
    """Konvertiert Metriken in Dictionary"""
    return {
        'uptime': {
            'seconds': self.uptime_seconds,
            'formatted': self.uptime_formatted,
        },
        'memory': {
            'current_mb': self.memory_current,
            'max_mb': self.memory_max,
            'min_mb': self.memory_min,
            'percent': self.memory_percent,
        },
        'cpu': {
            'current_percent': self.cpu_percent_current,
            'max_percent': self.cpu_percent_max,
            'min_percent': self.cpu_percent_min,
            'avg_percent': self.cpu_percent_avg,
            'time_seconds': self.cpu_time_seconds,
        },
        'disk': {
            'read_mb': self.disk_read_mb,
            'write_mb': self.disk_write_mb,
            'read_max_mb': self.disk_read_max,
            'read_min_mb': self.disk_read_min,
            'write_max_mb': self.disk_write_max,
            'write_min_mb': self.disk_write_min,
        },
        'network': {
            'sent_mb': self.network_sent_mb,
            'recv_mb': self.network_recv_mb,
            'sent_max_mb': self.network_sent_max,
            'sent_min_mb': self.network_sent_min,
            'recv_max_mb': self.network_recv_max,
            'recv_min_mb': self.network_recv_min,
        },
        'process': {
            'pid': self.process_id,
            'threads': self.threads,
            'open_files': self.open_files,
            'connections': self.connections,
        }
    }

Headers

Class representing HTTP headers with strongly typed common fields.

Source code in toolboxv2/utils/system/types.py

@dataclass
class Headers:
    """Class representing HTTP headers with strongly typed common fields."""
    # General Headers
    accept: None | str= None
    accept_charset: None | str= None
    accept_encoding: None | str= None
    accept_language: None | str= None
    accept_ranges: None | str= None
    access_control_allow_credentials: None | str= None
    access_control_allow_headers: None | str= None
    access_control_allow_methods: None | str= None
    access_control_allow_origin: None | str= None
    access_control_expose_headers: None | str= None
    access_control_max_age: None | str= None
    access_control_request_headers: None | str= None
    access_control_request_method: None | str= None
    age: None | str= None
    allow: None | str= None
    alt_svc: None | str= None
    authorization: None | str= None
    cache_control: None | str= None
    clear_site_data: None | str= None
    connection: None | str= None
    content_disposition: None | str= None
    content_encoding: None | str= None
    content_language: None | str= None
    content_length: None | str= None
    content_location: None | str= None
    content_range: None | str= None
    content_security_policy: None | str= None
    content_security_policy_report_only: None | str= None
    content_type: None | str= None
    cookie: None | str= None
    cross_origin_embedder_policy: None | str= None
    cross_origin_opener_policy: None | str= None
    cross_origin_resource_policy: None | str= None
    date: None | str= None
    device_memory: None | str= None
    digest: None | str= None
    dnt: None | str= None
    dpr: None | str= None
    etag: None | str= None
    expect: None | str= None
    expires: None | str= None
    feature_policy: None | str= None
    forwarded: None | str= None
    from_header: None | str= None  # 'from' is a Python keyword
    host: None | str= None
    if_match: None | str= None
    if_modified_since: None | str= None
    if_none_match: None | str= None
    if_range: None | str= None
    if_unmodified_since: None | str= None
    keep_alive: None | str= None
    large_allocation: None | str= None
    last_modified: None | str= None
    link: None | str= None
    location: None | str= None
    max_forwards: None | str= None
    origin: None | str= None
    pragma: None | str= None
    proxy_authenticate: None | str= None
    proxy_authorization: None | str= None
    public_key_pins: None | str= None
    public_key_pins_report_only: None | str= None
    range: None | str= None
    referer: None | str= None
    referrer_policy: None | str= None
    retry_after: None | str= None
    save_data: None | str= None
    sec_fetch_dest: None | str= None
    sec_fetch_mode: None | str= None
    sec_fetch_site: None | str= None
    sec_fetch_user: None | str= None
    sec_websocket_accept: None | str= None
    sec_websocket_extensions: None | str= None
    sec_websocket_key: None | str= None
    sec_websocket_protocol: None | str= None
    sec_websocket_version: None | str= None
    server: None | str= None
    server_timing: None | str= None
    service_worker_allowed: None | str= None
    set_cookie: None | str= None
    sourcemap: None | str= None
    strict_transport_security: None | str= None
    te: None | str= None
    timing_allow_origin: None | str= None
    tk: None | str= None
    trailer: None | str= None
    transfer_encoding: None | str= None
    upgrade: None | str= None
    upgrade_insecure_requests: None | str= None
    user_agent: None | str= None
    vary: None | str= None
    via: None | str= None
    warning: None | str= None
    www_authenticate: None | str= None
    x_content_type_options: None | str= None
    x_dns_prefetch_control: None | str= None
    x_forwarded_for: None | str= None
    x_forwarded_host: None | str= None
    x_forwarded_proto: None | str= None
    x_frame_options: None | str= None
    x_xss_protection: None | str= None

    # Browser-specific and custom headers
    sec_ch_ua: None | str= None
    sec_ch_ua_mobile: None | str= None
    sec_ch_ua_platform: None | str= None
    sec_ch_ua_arch: None | str= None
    sec_ch_ua_bitness: None | str= None
    sec_ch_ua_full_version: None | str= None
    sec_ch_ua_full_version_list: None | str= None
    sec_ch_ua_platform_version: None | str= None

    # HTMX specific headers
    hx_boosted: None | str= None
    hx_current_url: None | str= None
    hx_history_restore_request: None | str= None
    hx_prompt: None | str= None
    hx_request: None | str= None
    hx_target: None | str= None
    hx_trigger: None | str= None
    hx_trigger_name: None | str= None

    # Additional fields can be stored in extra_headers
    extra_headers: dict[str, str] = field(default_factory=dict)

    def __post_init__(self):
        """Convert header keys with hyphens to underscores for attribute access."""
        # Handle the 'from' header specifically since it's a Python keyword
        if 'from' in self.__dict__:
            self.from_header = self.__dict__.pop('from')

        # Store any attributes that weren't explicitly defined in extra_headers
        all_attrs = self.__annotations__.keys()
        for key in list(self.__dict__.keys()):
            if key not in all_attrs and key != "extra_headers":
                self.extra_headers[key.replace("_", "-")] = getattr(self, key)
                delattr(self, key)

    @classmethod
    def from_dict(cls, headers_dict: dict[str, str]) -> 'Headers':
        """Create a Headers instance from a dictionary."""
        # Convert header keys from hyphenated to underscore format for Python attributes
        processed_headers = {}
        extra_headers = {}

        for key, value in headers_dict.items():
            # Handle 'from' header specifically
            if key.lower() == 'from':
                processed_headers['from_header'] = value
                continue

            python_key = key.replace("-", "_").lower()
            if python_key in cls.__annotations__ and python_key != "extra_headers":
                processed_headers[python_key] = value
            else:
                extra_headers[key] = value

        return cls(**processed_headers, extra_headers=extra_headers)

    def to_dict(self) -> dict[str, str]:
        """Convert the Headers object back to a dictionary."""
        result = {}

        # Add regular attributes
        for key, value in self.__dict__.items():
            if key != "extra_headers" and value is not None:
                # Handle from_header specially
                if key == "from_header":
                    result["from"] = value
                else:
                    result[key.replace("_", "-")] = value

        # Add extra headers
        result.update(self.extra_headers)

        return result

__post_init__()

Convert header keys with hyphens to underscores for attribute access.

Source code in toolboxv2/utils/system/types.py

def __post_init__(self):
    """Convert header keys with hyphens to underscores for attribute access."""
    # Handle the 'from' header specifically since it's a Python keyword
    if 'from' in self.__dict__:
        self.from_header = self.__dict__.pop('from')

    # Store any attributes that weren't explicitly defined in extra_headers
    all_attrs = self.__annotations__.keys()
    for key in list(self.__dict__.keys()):
        if key not in all_attrs and key != "extra_headers":
            self.extra_headers[key.replace("_", "-")] = getattr(self, key)
            delattr(self, key)

from_dict(headers_dict) classmethod

Create a Headers instance from a dictionary.

Source code in toolboxv2/utils/system/types.py

@classmethod
def from_dict(cls, headers_dict: dict[str, str]) -> 'Headers':
    """Create a Headers instance from a dictionary."""
    # Convert header keys from hyphenated to underscore format for Python attributes
    processed_headers = {}
    extra_headers = {}

    for key, value in headers_dict.items():
        # Handle 'from' header specifically
        if key.lower() == 'from':
            processed_headers['from_header'] = value
            continue

        python_key = key.replace("-", "_").lower()
        if python_key in cls.__annotations__ and python_key != "extra_headers":
            processed_headers[python_key] = value
        else:
            extra_headers[key] = value

    return cls(**processed_headers, extra_headers=extra_headers)

to_dict()

Convert the Headers object back to a dictionary.

Source code in toolboxv2/utils/system/types.py

def to_dict(self) -> dict[str, str]:
    """Convert the Headers object back to a dictionary."""
    result = {}

    # Add regular attributes
    for key, value in self.__dict__.items():
        if key != "extra_headers" and value is not None:
            # Handle from_header specially
            if key == "from_header":
                result["from"] = value
            else:
                result[key.replace("_", "-")] = value

    # Add extra headers
    result.update(self.extra_headers)

    return result

MainToolType

Source code in toolboxv2/utils/system/types.py

class MainToolType:
    toolID: str
    app: A
    interface: ToolBoxInterfaces
    spec: str

    version: str
    tools: dict  # legacy
    name: str
    logger: logging
    color: str
    todo: Callable
    _on_exit: Callable
    stuf: bool
    config: dict
    user: U | None
    description: str

    @staticmethod
    def return_result(error: ToolBoxError = ToolBoxError.none,
                      exec_code: int = 0,
                      help_text: str = "",
                      data_info=None,
                      data=None,
                      data_to=None) -> Result:
        """proxi attr"""

    def load(self):
        """proxi attr"""

    def print(self, message, end="\n", **kwargs):
        """proxi attr"""

    def add_str_to_config(self, command):
        if len(command) != 2:
            self.logger.error('Invalid command must be key value')
            return False
        self.config[command[0]] = command[1]

    def webInstall(self, user_instance, construct_render) -> str:
        """"Returns a web installer for the given user instance and construct render template"""

    async def get_user(self, username: str) -> Result:
        return self.app.a_run_any(CLOUDM_AUTHMANAGER.GET_USER_BY_NAME, username=username, get_results=True)

load()

proxi attr

Source code in toolboxv2/utils/system/types.py

def load(self):
    """proxi attr"""

print(message, end='\n', **kwargs)

proxi attr

Source code in toolboxv2/utils/system/types.py

def print(self, message, end="\n", **kwargs):
    """proxi attr"""

return_result(error=ToolBoxError.none, exec_code=0, help_text='', data_info=None, data=None, data_to=None) staticmethod

proxi attr

Source code in toolboxv2/utils/system/types.py

@staticmethod
def return_result(error: ToolBoxError = ToolBoxError.none,
                  exec_code: int = 0,
                  help_text: str = "",
                  data_info=None,
                  data=None,
                  data_to=None) -> Result:
    """proxi attr"""

webInstall(user_instance, construct_render)

"Returns a web installer for the given user instance and construct render template

Source code in toolboxv2/utils/system/types.py

def webInstall(self, user_instance, construct_render) -> str:
    """"Returns a web installer for the given user instance and construct render template"""

Request

Class representing an HTTP request.

Source code in toolboxv2/utils/system/types.py

@dataclass
class Request:
    """Class representing an HTTP request."""
    content_type: str
    headers: Headers
    method: str
    path: str
    query_params: dict[str, Any] = field(default_factory=dict)
    form_data: dict[str, Any] | None = None
    body: Any | None = None

    @classmethod
    def from_dict(cls, data: dict[str, Any]) -> 'Request':
        """Create a Request instance from a dictionary."""
        headers = Headers.from_dict(data.get('headers', {}))

        # Extract other fields
        return cls(
            content_type=data.get('content_type', ''),
            headers=headers,
            method=data.get('method', ''),
            path=data.get('path', ''),
            query_params=data.get('query_params', {}),
            form_data=data.get('form_data'),
            body=data.get('body')
        )

    def to_dict(self) -> dict[str, Any]:
        """Convert the Request object back to a dictionary."""
        result = {
            'content_type': self.content_type,
            'headers': self.headers.to_dict(),
            'method': self.method,
            'path': self.path,
            'query_params': self.query_params,
        }

        if self.form_data is not None:
            result['form_data'] = self.form_data

        if self.body is not None:
            result['body'] = self.body

        return result

from_dict(data) classmethod

Create a Request instance from a dictionary.

Source code in toolboxv2/utils/system/types.py

@classmethod
def from_dict(cls, data: dict[str, Any]) -> 'Request':
    """Create a Request instance from a dictionary."""
    headers = Headers.from_dict(data.get('headers', {}))

    # Extract other fields
    return cls(
        content_type=data.get('content_type', ''),
        headers=headers,
        method=data.get('method', ''),
        path=data.get('path', ''),
        query_params=data.get('query_params', {}),
        form_data=data.get('form_data'),
        body=data.get('body')
    )

to_dict()

Convert the Request object back to a dictionary.

Source code in toolboxv2/utils/system/types.py

def to_dict(self) -> dict[str, Any]:
    """Convert the Request object back to a dictionary."""
    result = {
        'content_type': self.content_type,
        'headers': self.headers.to_dict(),
        'method': self.method,
        'path': self.path,
        'query_params': self.query_params,
    }

    if self.form_data is not None:
        result['form_data'] = self.form_data

    if self.body is not None:
        result['body'] = self.body

    return result

RequestData

Main class representing the complete request data structure.

Source code in toolboxv2/utils/system/types.py

@dataclass
class RequestData:
    """Main class representing the complete request data structure."""
    request: Request
    session: Session
    session_id: str

    @classmethod
    def from_dict(cls, data: dict[str, Any]) -> 'RequestData':
        """Create a RequestData instance from a dictionary."""
        return cls(
            request=Request.from_dict(data.get('request', {})),
            session=Session.from_dict(data.get('session', {})),
            session_id=data.get('session_id', '')
        )

    def to_dict(self) -> dict[str, Any]:
        """Convert the RequestData object back to a dictionary."""
        return {
            'request': self.request.to_dict(),
            'session': self.session.to_dict(),
            'session_id': self.session_id
        }

    def __getattr__(self, name: str) -> Any:
        """Delegate unknown attributes to the `request` object."""
        # Nur wenn das Attribut nicht direkt in RequestData existiert
        # und auch nicht `session` oder `session_id` ist
        if hasattr(self.request, name):
            return getattr(self.request, name)
        raise AttributeError(f"'RequestData' object has no attribute '{name}'")

    @classmethod
    def moc(cls):
        return cls(
            request=Request.from_dict({
                'content_type': 'application/x-www-form-urlencoded',
                'headers': {
                    'accept': '*/*',
                    'accept-encoding': 'gzip, deflate, br, zstd',
                    'accept-language': 'de-DE,de;q=0.9,en-US;q=0.8,en;q=0.7',
                    'connection': 'keep-alive',
                    'content-length': '107',
                    'content-type': 'application/x-www-form-urlencoded',
                    'cookie': 'session=abc123',
                    'host': 'localhost:8080',
                    'hx-current-url': 'http://localhost:8080/api/TruthSeeker/get_main_ui',
                    'hx-request': 'true',
                    'hx-target': 'estimates-guest_1fc2c9',
                    'hx-trigger': 'config-form-guest_1fc2c9',
                    'origin': 'http://localhost:8080',
                    'referer': 'http://localhost:8080/api/TruthSeeker/get_main_ui',
                    'sec-ch-ua': '"Chromium";v="134", "Not:A-Brand";v="24", "Google Chrome";v="134"',
                    'sec-ch-ua-mobile': '?0',
                    'sec-ch-ua-platform': '"Windows"',
                    'sec-fetch-dest': 'empty',
                    'sec-fetch-mode': 'cors',
                    'sec-fetch-site': 'same-origin',
                    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
                },
                'method': 'POST',
                'path': '/api/TruthSeeker/update_estimates',
                'query_params': {},
                'form_data': {
                    'param1': 'value1',
                    'param2': 'value2'
                }
            }),
            session=Session.from_dict({
                'SiID': '29a2e258e18252e2afd5ff943523f09c82f1bb9adfe382a6f33fc6a8381de898',
                'level': '1',
                'spec': '74eed1c8de06886842e235486c3c2fd6bcd60586998ac5beb87f13c0d1750e1d',
                'user_name': 'root',
                'custom_field': 'custom_value'
            }),
            session_id='0x29dd1ac0d1e30d3f'
        )

__getattr__(name)

Delegate unknown attributes to the request object.

Source code in toolboxv2/utils/system/types.py

def __getattr__(self, name: str) -> Any:
    """Delegate unknown attributes to the `request` object."""
    # Nur wenn das Attribut nicht direkt in RequestData existiert
    # und auch nicht `session` oder `session_id` ist
    if hasattr(self.request, name):
        return getattr(self.request, name)
    raise AttributeError(f"'RequestData' object has no attribute '{name}'")

from_dict(data) classmethod

Create a RequestData instance from a dictionary.

Source code in toolboxv2/utils/system/types.py

@classmethod
def from_dict(cls, data: dict[str, Any]) -> 'RequestData':
    """Create a RequestData instance from a dictionary."""
    return cls(
        request=Request.from_dict(data.get('request', {})),
        session=Session.from_dict(data.get('session', {})),
        session_id=data.get('session_id', '')
    )

to_dict()

Convert the RequestData object back to a dictionary.

Source code in toolboxv2/utils/system/types.py

def to_dict(self) -> dict[str, Any]:
    """Convert the RequestData object back to a dictionary."""
    return {
        'request': self.request.to_dict(),
        'session': self.session.to_dict(),
        'session_id': self.session_id
    }

Result

Source code in toolboxv2/utils/system/types.py

class Result(Generic[T]):
    _task = None
    _generic_type: Optional[Type] = None

    def __init__(self,
                 error: ToolBoxError,
                 result: ToolBoxResult,
                 info: ToolBoxInfo,
                 origin: Any | None = None,
                 generic_type: Optional[Type] = None
                 ):
        self.error: ToolBoxError = error
        self.result: ToolBoxResult = result
        self.info: ToolBoxInfo = info
        self.origin = origin
        self._generic_type = generic_type

    def __class_getitem__(cls, item):
        """Enable Result[Type] syntax"""

        class TypedResult(cls):
            _generic_type = item

            def __init__(self, *args, **kwargs):
                super().__init__(*args, **kwargs)
                self._generic_type = item

        return TypedResult

    def typed_get(self, key=None, default=None) -> T:
        """Get data with type validation"""
        data = self.get(key, default)

        if self._generic_type and data is not None:
            # Validate type matches generic parameter
            if not self._validate_type(data, self._generic_type):
                from toolboxv2 import get_logger
                get_logger().warning(f"Type mismatch: expected {self._generic_type}, got {type(data)}")

        return data

    async def typed_aget(self, key=None, default=None) -> T:
        """Async get data with type validation"""
        data = await self.aget(key, default)

        if self._generic_type and data is not None:
            if not self._validate_type(data, self._generic_type):
                from toolboxv2 import get_logger
                get_logger().warning(f"Type mismatch: expected {self._generic_type}, got {type(data)}")

        return data

    def _validate_type(self, data, expected_type) -> bool:
        """Validate data matches expected type"""
        try:
            # Handle List[Type] syntax
            origin = get_origin(expected_type)
            if origin is list or origin is List:
                if not isinstance(data, list):
                    return False

                # Check list element types if specified
                args = get_args(expected_type)
                if args and data:
                    element_type = args[0]
                    return all(isinstance(item, element_type) for item in data)
                return True

            # Handle other generic types
            elif origin is not None:
                return isinstance(data, origin)

            # Handle regular types
            else:
                return isinstance(data, expected_type)

        except Exception:
            return True  # Skip validation on error

    @classmethod
    def typed_ok(cls, data: T, data_info="", info="OK", interface=ToolBoxInterfaces.native) -> 'Result[T]':
        """Create OK result with type information"""
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=0, help_text=info)
        result = ToolBoxResult(data_to=interface, data=data, data_info=data_info, data_type=type(data).__name__)

        instance = cls(error=error, info=info_obj, result=result)
        if hasattr(cls, '_generic_type'):
            instance._generic_type = cls._generic_type

        return instance

    @classmethod
    def typed_json(cls, data: T, info="OK", interface=ToolBoxInterfaces.remote, exec_code=0,
                   status_code=None) -> 'Result[T]':
        """Create JSON result with type information"""
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=status_code or exec_code, help_text=info)

        result = ToolBoxResult(
            data_to=interface,
            data=data,
            data_info="JSON response",
            data_type="json"
        )

        instance = cls(error=error, info=info_obj, result=result)
        if hasattr(cls, '_generic_type'):
            instance._generic_type = cls._generic_type

        return instance

    def cast_to(self, target_type: Type[T]) -> 'Result[T]':
        """Cast result to different type"""
        new_result = Result(
            error=self.error,
            result=self.result,
            info=self.info,
            origin=self.origin,
            generic_type=target_type
        )
        new_result._generic_type = target_type
        return new_result

    def get_type_info(self) -> Optional[Type]:
        """Get the generic type information"""
        return self._generic_type

    def is_typed(self) -> bool:
        """Check if result has type information"""
        return self._generic_type is not None

    def as_result(self):
        return self

    def as_dict(self):
        return {
            "error":self.error.value if isinstance(self.error, Enum) else self.error,
        "result" : {
            "data_to":self.result.data_to.value if isinstance(self.result.data_to, Enum) else self.result.data_to,
            "data_info":self.result.data_info,
            "data":self.result.data,
            "data_type":self.result.data_type
        } if self.result else None,
        "info" : {
            "exec_code" : self.info.exec_code,  # exec_code umwandel in http resposn codes
        "help_text" : self.info.help_text
        } if self.info else None,
        "origin" : self.origin
        }

    def set_origin(self, origin):
        if self.origin is not None:
            raise ValueError("You cannot Change the origin of a Result!")
        self.origin = origin
        return self

    def set_dir_origin(self, name, extras="assets/"):
        if self.origin is not None:
            raise ValueError("You cannot Change the origin of a Result!")
        self.origin = f"mods/{name}/{extras}"
        return self

    def is_error(self):
        if _test_is_result(self.result.data):
            return self.result.data.is_error()
        if self.error == ToolBoxError.none:
            return False
        if self.info.exec_code == 0:
            return False
        return self.info.exec_code != 200

    def is_ok(self):
        return not self.is_error()

    def is_data(self):
        return self.result.data is not None

    def to_api_result(self):
        # print(f" error={self.error}, result= {self.result}, info= {self.info}, origin= {self.origin}")
        return ApiResult(
            error=self.error.value if isinstance(self.error, Enum) else self.error,
            result=ToolBoxResultBM(
                data_to=self.result.data_to.value if isinstance(self.result.data_to, Enum) else self.result.data_to,
                data_info=self.result.data_info,
                data=self.result.data,
                data_type=self.result.data_type
            ) if self.result else None,
            info=ToolBoxInfoBM(
                exec_code=self.info.exec_code,  # exec_code umwandel in http resposn codes
                help_text=self.info.help_text
            ) if self.info else None,
            origin=self.origin
        )

    def task(self, task):
        self._task = task
        return self

    @staticmethod
    def result_from_dict(error: str, result: dict, info: dict, origin: list or None or str):
        # print(f" error={self.error}, result= {self.result}, info= {self.info}, origin= {self.origin}")
        return ApiResult(
            error=error if isinstance(error, Enum) else error,
            result=ToolBoxResultBM(
                data_to=result.get('data_to') if isinstance(result.get('data_to'), Enum) else result.get('data_to'),
                data_info=result.get('data_info', '404'),
                data=result.get('data'),
                data_type=result.get('data_type', '404'),
            ) if result else ToolBoxResultBM(
                data_to=ToolBoxInterfaces.cli.value,
                data_info='',
                data='404',
                data_type='404',
            ),
            info=ToolBoxInfoBM(
                exec_code=info.get('exec_code', 404),
                help_text=info.get('help_text', '404')
            ) if info else ToolBoxInfoBM(
                exec_code=404,
                help_text='404'
            ),
            origin=origin
        ).as_result()

    @classmethod
    def stream(cls,
               stream_generator: Any,  # Renamed from source for clarity
               content_type: str = "text/event-stream",  # Default to SSE
               headers: dict | None = None,
               info: str = "OK",
               interface: ToolBoxInterfaces = ToolBoxInterfaces.remote,
               cleanup_func: Callable[[], None] | Callable[[], T] | Callable[[], AsyncGenerator[T, None]] | None = None):
        """
        Create a streaming response Result. Handles SSE and other stream types.

        Args:
            stream_generator: Any stream source (async generator, sync generator, iterable, or single item).
            content_type: Content-Type header (default: text/event-stream for SSE).
            headers: Additional HTTP headers for the response.
            info: Help text for the result.
            interface: Interface to send data to.
            cleanup_func: Optional function for cleanup.

        Returns:
            A Result object configured for streaming.
        """
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=0, help_text=info)

        final_generator: AsyncGenerator[str, None]

        if content_type == "text/event-stream":
            # For SSE, always use SSEGenerator.create_sse_stream to wrap the source.
            # SSEGenerator.create_sse_stream handles various types of stream_generator internally.
            final_generator = SSEGenerator.create_sse_stream(source=stream_generator, cleanup_func=cleanup_func)

            # Standard SSE headers for the HTTP response itself
            # These will be stored in the Result object. Rust side decides how to use them.
            standard_sse_headers = {
                "Cache-Control": "no-cache",  # SSE specific
                "Connection": "keep-alive",  # SSE specific
                "X-Accel-Buffering": "no",  # Useful for proxies with SSE
                # Content-Type is implicitly text/event-stream, will be in streaming_data below
            }
            all_response_headers = standard_sse_headers.copy()
            if headers:
                all_response_headers.update(headers)
        else:
            # For non-SSE streams.
            # If stream_generator is sync, wrap it to be async.
            # If already async or single item, it will be handled.
            # Rust's stream_generator in ToolboxClient seems to handle both sync/async Python generators.
            # For consistency with how SSEGenerator does it, we can wrap sync ones.
            if inspect.isgenerator(stream_generator) or \
                (not isinstance(stream_generator, str) and hasattr(stream_generator, '__iter__')):
                final_generator = SSEGenerator.wrap_sync_generator(stream_generator)  # Simple async wrapper
            elif inspect.isasyncgen(stream_generator):
                final_generator = stream_generator
            else:  # Single item or string
                async def _single_item_gen():
                    yield stream_generator

                final_generator = _single_item_gen()
            all_response_headers = headers if headers else {}

        # Prepare streaming data to be stored in the Result object
        streaming_data = {
            "type": "stream",  # Indicator for Rust side
            "generator": final_generator,
            "content_type": content_type,  # Let Rust know the intended content type
            "headers": all_response_headers  # Intended HTTP headers for the overall response
        }

        result_payload = ToolBoxResult(
            data_to=interface,
            data=streaming_data,
            data_info="Streaming response" if content_type != "text/event-stream" else "SSE Event Stream",
            data_type="stream"  # Generic type for Rust to identify it needs to stream from 'generator'
        )

        return cls(error=error, info=info_obj, result=result_payload)

    @classmethod
    def sse(cls,
            stream_generator: Any,
            info: str = "OK",
            interface: ToolBoxInterfaces = ToolBoxInterfaces.remote,
            cleanup_func: Callable[[], None] | Callable[[], T] | Callable[[], AsyncGenerator[T, None]] | None = None,
            # http_headers: Optional[dict] = None # If we want to allow overriding default SSE HTTP headers
            ):
        """
        Create an Server-Sent Events (SSE) streaming response Result.

        Args:
            stream_generator: A source yielding individual data items. This can be an
                              async generator, sync generator, iterable, or a single item.
                              Each item will be formatted as an SSE event.
            info: Optional help text for the Result.
            interface: Optional ToolBoxInterface to target.
            cleanup_func: Optional cleanup function to run when the stream ends or is cancelled.
            #http_headers: Optional dictionary of custom HTTP headers for the SSE response.

        Returns:
            A Result object configured for SSE streaming.
        """
        # Result.stream will handle calling SSEGenerator.create_sse_stream
        # and setting appropriate default headers for SSE when content_type is "text/event-stream".
        return cls.stream(
            stream_generator=stream_generator,
            content_type="text/event-stream",
            # headers=http_headers, # Pass if we add http_headers param
            info=info,
            interface=interface,
            cleanup_func=cleanup_func
        )

    @classmethod
    def default(cls, interface=ToolBoxInterfaces.native):
        error = ToolBoxError.none
        info = ToolBoxInfo(exec_code=-1, help_text="")
        result = ToolBoxResult(data_to=interface)
        return cls(error=error, info=info, result=result)

    @classmethod
    def json(cls, data, info="OK", interface=ToolBoxInterfaces.remote, exec_code=0, status_code=None):
        """Create a JSON response Result."""
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=status_code or exec_code, help_text=info)

        result = ToolBoxResult(
            data_to=interface,
            data=data,
            data_info="JSON response",
            data_type="json"
        )

        return cls(error=error, info=info_obj, result=result)

    @classmethod
    def text(cls, text_data, content_type="text/plain",exec_code=None,status=200, info="OK", interface=ToolBoxInterfaces.remote, headers=None):
        """Create a text response Result with specific content type."""
        if headers is not None:
            return cls.html(text_data, status= exec_code or status, info=info, headers=headers)
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=exec_code or status, help_text=info)

        result = ToolBoxResult(
            data_to=interface,
            data=text_data,
            data_info="Text response",
            data_type=content_type
        )

        return cls(error=error, info=info_obj, result=result)

    @classmethod
    def binary(cls, data, content_type="application/octet-stream", download_name=None, info="OK",
               interface=ToolBoxInterfaces.remote):
        """Create a binary data response Result."""
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=0, help_text=info)

        # Create a dictionary with binary data and metadata
        binary_data = {
            "data": data,
            "content_type": content_type,
            "filename": download_name
        }

        result = ToolBoxResult(
            data_to=interface,
            data=binary_data,
            data_info=f"Binary response: {download_name}" if download_name else "Binary response",
            data_type="binary"
        )

        return cls(error=error, info=info_obj, result=result)

    @classmethod
    def file(cls, data, filename, content_type=None, info="OK", interface=ToolBoxInterfaces.remote):
        """Create a file download response Result.

        Args:
            data: File data as bytes or base64 string
            filename: Name of the file for download
            content_type: MIME type of the file (auto-detected if None)
            info: Response info text
            interface: Target interface

        Returns:
            Result object configured for file download
        """
        import base64
        import mimetypes

        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=200, help_text=info)

        # Auto-detect content type if not provided
        if content_type is None:
            content_type, _ = mimetypes.guess_type(filename)
            if content_type is None:
                content_type = "application/octet-stream"

        # Ensure data is base64 encoded string (as expected by Rust server)
        if isinstance(data, bytes):
            base64_data = base64.b64encode(data).decode('utf-8')
        elif isinstance(data, str):
            # Assume it's already base64 encoded
            base64_data = data
        else:
            raise ValueError("File data must be bytes or base64 string")

        result = ToolBoxResult(
            data_to=interface,
            data=base64_data,  # Rust expects base64 string for "file" type
            data_info=f"File download: {filename}",
            data_type="file"
        )

        return cls(error=error, info=info_obj, result=result)

    @classmethod
    def redirect(cls, url, status_code=302, info="Redirect", interface=ToolBoxInterfaces.remote):
        """Create a redirect response."""
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=status_code, help_text=info)

        result = ToolBoxResult(
            data_to=interface,
            data=url,
            data_info="Redirect response",
            data_type="redirect"
        )

        return cls(error=error, info=info_obj, result=result)

    @classmethod
    def ok(cls, data=None, data_info="", info="OK", interface=ToolBoxInterfaces.native):
        error = ToolBoxError.none
        info = ToolBoxInfo(exec_code=0, help_text=info)
        result = ToolBoxResult(data_to=interface, data=data, data_info=data_info, data_type=type(data).__name__)
        return cls(error=error, info=info, result=result)

    @classmethod
    def html(cls, data=None, data_info="", info="OK", interface=ToolBoxInterfaces.remote, data_type="html",status=200, headers=None, row=False):
        error = ToolBoxError.none
        info = ToolBoxInfo(exec_code=status, help_text=info)
        from ...utils.system.getting_and_closing_app import get_app

        if not row and not '"<div class="main-content""' in data:
            data = f'<div class="main-content frosted-glass">{data}<div>'
        if not row and not get_app().web_context() in data:
            data = get_app().web_context() + data

        if isinstance(headers, dict):
            result = ToolBoxResult(data_to=interface, data={'html':data,'headers':headers}, data_info=data_info,
                                   data_type="special_html")
        else:
            result = ToolBoxResult(data_to=interface, data=data, data_info=data_info,
                                   data_type=data_type if data_type is not None else type(data).__name__)
        return cls(error=error, info=info, result=result)

    @classmethod
    def future(cls, data=None, data_info="", info="OK", interface=ToolBoxInterfaces.future):
        error = ToolBoxError.none
        info = ToolBoxInfo(exec_code=0, help_text=info)
        result = ToolBoxResult(data_to=interface, data=data, data_info=data_info, data_type="future")
        return cls(error=error, info=info, result=result)

    @classmethod
    def custom_error(cls, data=None, data_info="", info="", exec_code=-1, interface=ToolBoxInterfaces.native):
        error = ToolBoxError.custom_error
        info = ToolBoxInfo(exec_code=exec_code, help_text=info)
        result = ToolBoxResult(data_to=interface, data=data, data_info=data_info, data_type=type(data).__name__)
        return cls(error=error, info=info, result=result)

    @classmethod
    def error(cls, data=None, data_info="", info="", exec_code=450, interface=ToolBoxInterfaces.remote):
        error = ToolBoxError.custom_error
        info = ToolBoxInfo(exec_code=exec_code, help_text=info)
        result = ToolBoxResult(data_to=interface, data=data, data_info=data_info, data_type=type(data).__name__)
        return cls(error=error, info=info, result=result)

    @classmethod
    def default_user_error(cls, info="", exec_code=-3, interface=ToolBoxInterfaces.native, data=None):
        error = ToolBoxError.input_error
        info = ToolBoxInfo(exec_code, info)
        result = ToolBoxResult(data_to=interface, data=data, data_type=type(data).__name__)
        return cls(error=error, info=info, result=result)

    @classmethod
    def default_internal_error(cls, info="", exec_code=-2, interface=ToolBoxInterfaces.native, data=None):
        error = ToolBoxError.internal_error
        info = ToolBoxInfo(exec_code, info)
        result = ToolBoxResult(data_to=interface, data=data, data_type=type(data).__name__)
        return cls(error=error, info=info, result=result)

    def print(self, show=True, show_data=True, prifix="", full_data=False):
        data = '\n' + f"{((prifix + f'Data_{self.result.data_type}: ' + str(self.result.data) if self.result.data is not None else 'NO Data') if not isinstance(self.result.data, Result) else self.result.data.print(show=False, show_data=show_data, prifix=prifix + '-')) if show_data else 'Data: private'}"
        origin = '\n' + f"{prifix + 'Origin: ' + str(self.origin) if self.origin is not None else 'NO Origin'}"
        text = (f"Function Exec code: {self.info.exec_code}"
                f"\n{prifix}Info's:"
                f" {self.info.help_text} {'<|> ' + str(self.result.data_info) if self.result.data_info is not None else ''}"
                f"{origin}{((data[:100]+'...') if not full_data else (data)) if not data.endswith('NO Data') else ''}\n")
        if not show:
            return text
        print("\n======== Result ========\n" + text + "------- EndOfD -------")
        return self

    def log(self, show_data=True, prifix=""):
        from toolboxv2 import get_logger
        get_logger().debug(self.print(show=False, show_data=show_data, prifix=prifix).replace("\n", " - "))
        return self

    def __str__(self):
        return self.print(show=False, show_data=True)

    def get(self, key=None, default=None):
        data = self.result.data
        if isinstance(data, Result):
            return data.get(key=key, default=default)
        if key is not None and isinstance(data, dict):
            return data.get(key, default)
        return data if data is not None else default

    async def aget(self, key=None, default=None):
        if asyncio.isfuture(self.result.data) or asyncio.iscoroutine(self.result.data) or (
            isinstance(self.result.data_to, Enum) and self.result.data_to.name == ToolBoxInterfaces.future.name):
            data = await self.result.data
        else:
            data = self.get(key=None, default=None)
        if isinstance(data, Result):
            return data.get(key=key, default=default)
        if key is not None and isinstance(data, dict):
            return data.get(key, default)
        return data if data is not None else default

    def lazy_return(self, _=0, data=None, **kwargs):
        flags = ['raise', 'logg', 'user', 'intern']
        flag = flags[_] if isinstance(_, int) else _
        if self.info.exec_code == 0:
            return self if data is None else data if _test_is_result(data) else self.ok(data=data, **kwargs)
        if flag == 'raise':
            raise ValueError(self.print(show=False))
        if flag == 'logg':
            from .. import get_logger
            get_logger().error(self.print(show=False))

        if flag == 'user':
            return self if data is None else data if _test_is_result(data) else self.default_user_error(data=data,
                                                                                                        **kwargs)
        if flag == 'intern':
            return self if data is None else data if _test_is_result(data) else self.default_internal_error(data=data,
                                                                                                            **kwargs)

        return self if data is None else data if _test_is_result(data) else self.custom_error(data=data, **kwargs)

    @property
    def bg_task(self):
        return self._task

__class_getitem__(item)

Enable Result[Type] syntax

Source code in toolboxv2/utils/system/types.py

def __class_getitem__(cls, item):
    """Enable Result[Type] syntax"""

    class TypedResult(cls):
        _generic_type = item

        def __init__(self, *args, **kwargs):
            super().__init__(*args, **kwargs)
            self._generic_type = item

    return TypedResult

binary(data, content_type='application/octet-stream', download_name=None, info='OK', interface=ToolBoxInterfaces.remote) classmethod

Create a binary data response Result.

Source code in toolboxv2/utils/system/types.py

@classmethod
def binary(cls, data, content_type="application/octet-stream", download_name=None, info="OK",
           interface=ToolBoxInterfaces.remote):
    """Create a binary data response Result."""
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=0, help_text=info)

    # Create a dictionary with binary data and metadata
    binary_data = {
        "data": data,
        "content_type": content_type,
        "filename": download_name
    }

    result = ToolBoxResult(
        data_to=interface,
        data=binary_data,
        data_info=f"Binary response: {download_name}" if download_name else "Binary response",
        data_type="binary"
    )

    return cls(error=error, info=info_obj, result=result)

cast_to(target_type)

Cast result to different type

Source code in toolboxv2/utils/system/types.py

def cast_to(self, target_type: Type[T]) -> 'Result[T]':
    """Cast result to different type"""
    new_result = Result(
        error=self.error,
        result=self.result,
        info=self.info,
        origin=self.origin,
        generic_type=target_type
    )
    new_result._generic_type = target_type
    return new_result

file(data, filename, content_type=None, info='OK', interface=ToolBoxInterfaces.remote) classmethod

Create a file download response Result.

Parameters:

Name	Type	Description	Default
data		File data as bytes or base64 string	required
filename		Name of the file for download	required
content_type		MIME type of the file (auto-detected if None)	None
info		Response info text	'OK'
interface		Target interface	remote

Returns:

Type	Description
	Result object configured for file download

Source code in toolboxv2/utils/system/types.py

@classmethod
def file(cls, data, filename, content_type=None, info="OK", interface=ToolBoxInterfaces.remote):
    """Create a file download response Result.

    Args:
        data: File data as bytes or base64 string
        filename: Name of the file for download
        content_type: MIME type of the file (auto-detected if None)
        info: Response info text
        interface: Target interface

    Returns:
        Result object configured for file download
    """
    import base64
    import mimetypes

    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=200, help_text=info)

    # Auto-detect content type if not provided
    if content_type is None:
        content_type, _ = mimetypes.guess_type(filename)
        if content_type is None:
            content_type = "application/octet-stream"

    # Ensure data is base64 encoded string (as expected by Rust server)
    if isinstance(data, bytes):
        base64_data = base64.b64encode(data).decode('utf-8')
    elif isinstance(data, str):
        # Assume it's already base64 encoded
        base64_data = data
    else:
        raise ValueError("File data must be bytes or base64 string")

    result = ToolBoxResult(
        data_to=interface,
        data=base64_data,  # Rust expects base64 string for "file" type
        data_info=f"File download: {filename}",
        data_type="file"
    )

    return cls(error=error, info=info_obj, result=result)

get_type_info()

Get the generic type information

Source code in toolboxv2/utils/system/types.py

def get_type_info(self) -> Optional[Type]:
    """Get the generic type information"""
    return self._generic_type

is_typed()

Check if result has type information

Source code in toolboxv2/utils/system/types.py

def is_typed(self) -> bool:
    """Check if result has type information"""
    return self._generic_type is not None

json(data, info='OK', interface=ToolBoxInterfaces.remote, exec_code=0, status_code=None) classmethod

Create a JSON response Result.

Source code in toolboxv2/utils/system/types.py

@classmethod
def json(cls, data, info="OK", interface=ToolBoxInterfaces.remote, exec_code=0, status_code=None):
    """Create a JSON response Result."""
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=status_code or exec_code, help_text=info)

    result = ToolBoxResult(
        data_to=interface,
        data=data,
        data_info="JSON response",
        data_type="json"
    )

    return cls(error=error, info=info_obj, result=result)

redirect(url, status_code=302, info='Redirect', interface=ToolBoxInterfaces.remote) classmethod

Create a redirect response.

Source code in toolboxv2/utils/system/types.py

@classmethod
def redirect(cls, url, status_code=302, info="Redirect", interface=ToolBoxInterfaces.remote):
    """Create a redirect response."""
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=status_code, help_text=info)

    result = ToolBoxResult(
        data_to=interface,
        data=url,
        data_info="Redirect response",
        data_type="redirect"
    )

    return cls(error=error, info=info_obj, result=result)

sse(stream_generator, info='OK', interface=ToolBoxInterfaces.remote, cleanup_func=None) classmethod

Create an Server-Sent Events (SSE) streaming response Result.

Parameters:

Name	Type	Description	Default
stream_generator	Any	A source yielding individual data items. This can be an async generator, sync generator, iterable, or a single item. Each item will be formatted as an SSE event.	required
info	str	Optional help text for the Result.	'OK'
interface	ToolBoxInterfaces	Optional ToolBoxInterface to target.	remote
cleanup_func	Callable[[], None] | Callable[[], T] | Callable[[], AsyncGenerator[T, None]] | None	Optional cleanup function to run when the stream ends or is cancelled.	None
#http_headers		Optional dictionary of custom HTTP headers for the SSE response.	required

Returns:

Type	Description
	A Result object configured for SSE streaming.

Source code in toolboxv2/utils/system/types.py

@classmethod
def sse(cls,
        stream_generator: Any,
        info: str = "OK",
        interface: ToolBoxInterfaces = ToolBoxInterfaces.remote,
        cleanup_func: Callable[[], None] | Callable[[], T] | Callable[[], AsyncGenerator[T, None]] | None = None,
        # http_headers: Optional[dict] = None # If we want to allow overriding default SSE HTTP headers
        ):
    """
    Create an Server-Sent Events (SSE) streaming response Result.

    Args:
        stream_generator: A source yielding individual data items. This can be an
                          async generator, sync generator, iterable, or a single item.
                          Each item will be formatted as an SSE event.
        info: Optional help text for the Result.
        interface: Optional ToolBoxInterface to target.
        cleanup_func: Optional cleanup function to run when the stream ends or is cancelled.
        #http_headers: Optional dictionary of custom HTTP headers for the SSE response.

    Returns:
        A Result object configured for SSE streaming.
    """
    # Result.stream will handle calling SSEGenerator.create_sse_stream
    # and setting appropriate default headers for SSE when content_type is "text/event-stream".
    return cls.stream(
        stream_generator=stream_generator,
        content_type="text/event-stream",
        # headers=http_headers, # Pass if we add http_headers param
        info=info,
        interface=interface,
        cleanup_func=cleanup_func
    )

stream(stream_generator, content_type='text/event-stream', headers=None, info='OK', interface=ToolBoxInterfaces.remote, cleanup_func=None) classmethod

Create a streaming response Result. Handles SSE and other stream types.

Parameters:

Name	Type	Description	Default
stream_generator	Any	Any stream source (async generator, sync generator, iterable, or single item).	required
content_type	str	Content-Type header (default: text/event-stream for SSE).	'text/event-stream'
headers	dict | None	Additional HTTP headers for the response.	None
info	str	Help text for the result.	'OK'
interface	ToolBoxInterfaces	Interface to send data to.	remote
cleanup_func	Callable[[], None] | Callable[[], T] | Callable[[], AsyncGenerator[T, None]] | None	Optional function for cleanup.	None

Returns:

Type	Description
	A Result object configured for streaming.

Source code in toolboxv2/utils/system/types.py

@classmethod
def stream(cls,
           stream_generator: Any,  # Renamed from source for clarity
           content_type: str = "text/event-stream",  # Default to SSE
           headers: dict | None = None,
           info: str = "OK",
           interface: ToolBoxInterfaces = ToolBoxInterfaces.remote,
           cleanup_func: Callable[[], None] | Callable[[], T] | Callable[[], AsyncGenerator[T, None]] | None = None):
    """
    Create a streaming response Result. Handles SSE and other stream types.

    Args:
        stream_generator: Any stream source (async generator, sync generator, iterable, or single item).
        content_type: Content-Type header (default: text/event-stream for SSE).
        headers: Additional HTTP headers for the response.
        info: Help text for the result.
        interface: Interface to send data to.
        cleanup_func: Optional function for cleanup.

    Returns:
        A Result object configured for streaming.
    """
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=0, help_text=info)

    final_generator: AsyncGenerator[str, None]

    if content_type == "text/event-stream":
        # For SSE, always use SSEGenerator.create_sse_stream to wrap the source.
        # SSEGenerator.create_sse_stream handles various types of stream_generator internally.
        final_generator = SSEGenerator.create_sse_stream(source=stream_generator, cleanup_func=cleanup_func)

        # Standard SSE headers for the HTTP response itself
        # These will be stored in the Result object. Rust side decides how to use them.
        standard_sse_headers = {
            "Cache-Control": "no-cache",  # SSE specific
            "Connection": "keep-alive",  # SSE specific
            "X-Accel-Buffering": "no",  # Useful for proxies with SSE
            # Content-Type is implicitly text/event-stream, will be in streaming_data below
        }
        all_response_headers = standard_sse_headers.copy()
        if headers:
            all_response_headers.update(headers)
    else:
        # For non-SSE streams.
        # If stream_generator is sync, wrap it to be async.
        # If already async or single item, it will be handled.
        # Rust's stream_generator in ToolboxClient seems to handle both sync/async Python generators.
        # For consistency with how SSEGenerator does it, we can wrap sync ones.
        if inspect.isgenerator(stream_generator) or \
            (not isinstance(stream_generator, str) and hasattr(stream_generator, '__iter__')):
            final_generator = SSEGenerator.wrap_sync_generator(stream_generator)  # Simple async wrapper
        elif inspect.isasyncgen(stream_generator):
            final_generator = stream_generator
        else:  # Single item or string
            async def _single_item_gen():
                yield stream_generator

            final_generator = _single_item_gen()
        all_response_headers = headers if headers else {}

    # Prepare streaming data to be stored in the Result object
    streaming_data = {
        "type": "stream",  # Indicator for Rust side
        "generator": final_generator,
        "content_type": content_type,  # Let Rust know the intended content type
        "headers": all_response_headers  # Intended HTTP headers for the overall response
    }

    result_payload = ToolBoxResult(
        data_to=interface,
        data=streaming_data,
        data_info="Streaming response" if content_type != "text/event-stream" else "SSE Event Stream",
        data_type="stream"  # Generic type for Rust to identify it needs to stream from 'generator'
    )

    return cls(error=error, info=info_obj, result=result_payload)

text(text_data, content_type='text/plain', exec_code=None, status=200, info='OK', interface=ToolBoxInterfaces.remote, headers=None) classmethod

Create a text response Result with specific content type.

Source code in toolboxv2/utils/system/types.py

@classmethod
def text(cls, text_data, content_type="text/plain",exec_code=None,status=200, info="OK", interface=ToolBoxInterfaces.remote, headers=None):
    """Create a text response Result with specific content type."""
    if headers is not None:
        return cls.html(text_data, status= exec_code or status, info=info, headers=headers)
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=exec_code or status, help_text=info)

    result = ToolBoxResult(
        data_to=interface,
        data=text_data,
        data_info="Text response",
        data_type=content_type
    )

    return cls(error=error, info=info_obj, result=result)

typed_aget(key=None, default=None) async

Async get data with type validation

Source code in toolboxv2/utils/system/types.py

async def typed_aget(self, key=None, default=None) -> T:
    """Async get data with type validation"""
    data = await self.aget(key, default)

    if self._generic_type and data is not None:
        if not self._validate_type(data, self._generic_type):
            from toolboxv2 import get_logger
            get_logger().warning(f"Type mismatch: expected {self._generic_type}, got {type(data)}")

    return data

typed_get(key=None, default=None)

Get data with type validation

Source code in toolboxv2/utils/system/types.py

def typed_get(self, key=None, default=None) -> T:
    """Get data with type validation"""
    data = self.get(key, default)

    if self._generic_type and data is not None:
        # Validate type matches generic parameter
        if not self._validate_type(data, self._generic_type):
            from toolboxv2 import get_logger
            get_logger().warning(f"Type mismatch: expected {self._generic_type}, got {type(data)}")

    return data

typed_json(data, info='OK', interface=ToolBoxInterfaces.remote, exec_code=0, status_code=None) classmethod

Create JSON result with type information

Source code in toolboxv2/utils/system/types.py

@classmethod
def typed_json(cls, data: T, info="OK", interface=ToolBoxInterfaces.remote, exec_code=0,
               status_code=None) -> 'Result[T]':
    """Create JSON result with type information"""
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=status_code or exec_code, help_text=info)

    result = ToolBoxResult(
        data_to=interface,
        data=data,
        data_info="JSON response",
        data_type="json"
    )

    instance = cls(error=error, info=info_obj, result=result)
    if hasattr(cls, '_generic_type'):
        instance._generic_type = cls._generic_type

    return instance

typed_ok(data, data_info='', info='OK', interface=ToolBoxInterfaces.native) classmethod

Create OK result with type information

Source code in toolboxv2/utils/system/types.py

@classmethod
def typed_ok(cls, data: T, data_info="", info="OK", interface=ToolBoxInterfaces.native) -> 'Result[T]':
    """Create OK result with type information"""
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=0, help_text=info)
    result = ToolBoxResult(data_to=interface, data=data, data_info=data_info, data_type=type(data).__name__)

    instance = cls(error=error, info=info_obj, result=result)
    if hasattr(cls, '_generic_type'):
        instance._generic_type = cls._generic_type

    return instance

SSEGenerator

Production-ready SSE generator that converts any data source to properly formatted Server-Sent Events compatible with browsers.

Source code in toolboxv2/utils/system/types.py

class SSEGenerator:
    """
    Production-ready SSE generator that converts any data source to
    properly formatted Server-Sent Events compatible with browsers.
    """

    @staticmethod
    def format_sse_event(data: Any) -> str:
        """Format any data as a proper SSE event message."""
        # Already formatted as SSE
        if isinstance(data, str) and (data.startswith('data:') or data.startswith('event:')) and '\n\n' in data:
            return data

        # Handle bytes (binary data)
        if isinstance(data, bytes):
            try:
                # Try to decode as UTF-8 first
                decoded_data_str = data.decode('utf-8')
                # If decoding works, treat it as a string for further processing
                # This allows binary data that is valid UTF-8 JSON to be processed as JSON.
                data = decoded_data_str
            except UnicodeDecodeError:
                # Binary data that is not UTF-8, encode as base64
                b64_data = base64.b64encode(data).decode('utf-8')
                return f"event: binary\ndata: {b64_data}\n\n"

        # Convert non-string objects (that are not already bytes) to JSON string
        # If data was bytes and successfully decoded to UTF-8 string, it will be processed here.
        original_data_type_was_complex = False
        if not isinstance(data, str):
            original_data_type_was_complex = True
            try:
                data_str = json.dumps(data)
            except Exception:
                data_str = str(data)  # Fallback to string representation
        else:
            data_str = data  # data is already a string

        # Handle JSON data with special event formatting
        # data_str now holds the string representation (either original string or JSON string)
        if data_str.strip().startswith('{'):
            try:
                json_data = json.loads(data_str)
                if isinstance(json_data, dict) and 'event' in json_data:
                    event_type = json_data['event']
                    event_id = json_data.get('id', None)  # Use None to distinguish from empty string

                    # Determine the actual data payload for the SSE 'data:' field
                    # If 'data' key exists in json_data, use its content.
                    # Otherwise, use the original data_str (which is the JSON of json_data).
                    if 'data' in json_data:
                        payload_content = json_data['data']
                        # If payload_content is complex, re-serialize it to JSON string
                        if isinstance(payload_content, dict | list):
                            sse_data_field = json.dumps(payload_content)
                        else:  # Simple type (string, number, bool)
                            sse_data_field = str(payload_content)
                    else:
                        # If original data was complex (e.g. dict) and became json_data,
                        # and no 'data' key in it, then use the full json_data as payload.
                        # If original data was a simple string that happened to be JSON parsable
                        # but without 'event' key, it would have been handled by "Regular JSON without event"
                        # or "Plain text" later.
                        # This path implies original data was a dict with 'event' key.
                        sse_data_field = data_str

                    sse_lines = []
                    if event_type:  # Should always be true here
                        sse_lines.append(f"event: {event_type}")
                    if event_id is not None:  # Check for None, allow empty string id
                        sse_lines.append(f"id: {event_id}")

                    # Handle multi-line data for the data field
                    for line in sse_data_field.splitlines():
                        sse_lines.append(f"data: {line}")

                    return "\n".join(sse_lines) + "\n\n"
                else:
                    # Regular JSON without special 'event' key
                    sse_lines = []
                    for line in data_str.splitlines():
                        sse_lines.append(f"data: {line}")
                    return "\n".join(sse_lines) + "\n\n"
            except json.JSONDecodeError:
                # Not valid JSON, treat as plain text
                sse_lines = []
                for line in data_str.splitlines():
                    sse_lines.append(f"data: {line}")
                return "\n".join(sse_lines) + "\n\n"
        else:
            # Plain text
            sse_lines = []
            for line in data_str.splitlines():
                sse_lines.append(f"data: {line}")
            return "\n".join(sse_lines) + "\n\n"

    @classmethod
    async def wrap_sync_generator(cls, generator):
        """Convert a synchronous generator to an async generator."""
        for item in generator:
            yield item
            # Allow other tasks to run
            await asyncio.sleep(0)

    @classmethod
    async def create_sse_stream(
        cls,
        source: Any,  # Changed from positional arg to keyword for clarity in Result.stream
        cleanup_func: Callable[[], None] | Callable[[], T] | Callable[[], AsyncGenerator[T, None]] | None = None
    ) -> AsyncGenerator[str, None]:
        """
        Convert any source to a properly formatted SSE stream.

        Args:
            source: Can be async generator, sync generator, iterable, or a single item.
            cleanup_func: Optional function to call when the stream ends or is cancelled.
                          Can be a synchronous function, async function, or async generator.

        Yields:
            Properly formatted SSE messages (strings).
        """
        # Send stream start event
        # This structure ensures data field contains {"id":"0"}
        yield cls.format_sse_event({"event": "stream_start", "data": {"id": "0"}})

        try:
            # Handle different types of sources
            if inspect.isasyncgen(source):
                # Source is already an async generator
                async for item in source:
                    yield cls.format_sse_event(item)
            elif inspect.isgenerator(source) or (not isinstance(source, str) and hasattr(source, '__iter__')):
                # Source is a sync generator or iterable (but not a string)
                # Strings are iterable but should be treated as single items unless explicitly made a generator
                async for item in cls.wrap_sync_generator(source):
                    yield cls.format_sse_event(item)
            else:
                # Single item (including strings)
                yield cls.format_sse_event(source)
        except asyncio.CancelledError:
            # Client disconnected
            yield cls.format_sse_event({"event": "cancelled", "data": {"id": "cancelled"}})
            raise
        except Exception as e:
            # Error in stream
            error_info = {
                "event": "error",
                "data": {  # Ensure payload is under 'data' key for the new format_sse_event logic
                    "message": str(e),
                    "traceback": traceback.format_exc()
                }
            }
            yield cls.format_sse_event(error_info)
        finally:
            # Always send end event
            yield cls.format_sse_event({"event": "stream_end", "data": {"id": "final"}})

            # Execute cleanup function if provided
            if cleanup_func:
                try:
                    if inspect.iscoroutinefunction(cleanup_func):  # Check if it's an async def function
                        await cleanup_func()
                    elif inspect.isasyncgenfunction(cleanup_func) or inspect.isasyncgen(
                        cleanup_func):  # Check if it's an async def generator function or already an async generator
                        # If it's a function, call it to get the generator
                        gen_to_exhaust = cleanup_func() if inspect.isasyncgenfunction(cleanup_func) else cleanup_func
                        async for _ in gen_to_exhaust:
                            pass  # Exhaust the generator to ensure cleanup completes
                    else:
                        # Synchronous function
                        cleanup_func()
                except Exception as e:
                    # Log cleanup errors but don't propagate them to client
                    error_info_cleanup = {
                        "event": "cleanup_error",
                        "data": {  # Ensure payload is under 'data' key
                            "message": str(e),
                            "traceback": traceback.format_exc()
                        }
                    }
                    # We can't yield here as the stream is already closing/closed.
                    # Instead, log the error.
                    # In a real app, use a proper logger.
                    print(f"SSE cleanup error: {cls.format_sse_event(error_info_cleanup)}", flush=True)

create_sse_stream(source, cleanup_func=None) async classmethod

Convert any source to a properly formatted SSE stream.

Parameters:

Name	Type	Description	Default
source	Any	Can be async generator, sync generator, iterable, or a single item.	required
cleanup_func	Callable[[], None] | Callable[[], T] | Callable[[], AsyncGenerator[T, None]] | None	Optional function to call when the stream ends or is cancelled. Can be a synchronous function, async function, or async generator.	None

Yields:

Type	Description
AsyncGenerator[str, None]	Properly formatted SSE messages (strings).

Source code in toolboxv2/utils/system/types.py

@classmethod
async def create_sse_stream(
    cls,
    source: Any,  # Changed from positional arg to keyword for clarity in Result.stream
    cleanup_func: Callable[[], None] | Callable[[], T] | Callable[[], AsyncGenerator[T, None]] | None = None
) -> AsyncGenerator[str, None]:
    """
    Convert any source to a properly formatted SSE stream.

    Args:
        source: Can be async generator, sync generator, iterable, or a single item.
        cleanup_func: Optional function to call when the stream ends or is cancelled.
                      Can be a synchronous function, async function, or async generator.

    Yields:
        Properly formatted SSE messages (strings).
    """
    # Send stream start event
    # This structure ensures data field contains {"id":"0"}
    yield cls.format_sse_event({"event": "stream_start", "data": {"id": "0"}})

    try:
        # Handle different types of sources
        if inspect.isasyncgen(source):
            # Source is already an async generator
            async for item in source:
                yield cls.format_sse_event(item)
        elif inspect.isgenerator(source) or (not isinstance(source, str) and hasattr(source, '__iter__')):
            # Source is a sync generator or iterable (but not a string)
            # Strings are iterable but should be treated as single items unless explicitly made a generator
            async for item in cls.wrap_sync_generator(source):
                yield cls.format_sse_event(item)
        else:
            # Single item (including strings)
            yield cls.format_sse_event(source)
    except asyncio.CancelledError:
        # Client disconnected
        yield cls.format_sse_event({"event": "cancelled", "data": {"id": "cancelled"}})
        raise
    except Exception as e:
        # Error in stream
        error_info = {
            "event": "error",
            "data": {  # Ensure payload is under 'data' key for the new format_sse_event logic
                "message": str(e),
                "traceback": traceback.format_exc()
            }
        }
        yield cls.format_sse_event(error_info)
    finally:
        # Always send end event
        yield cls.format_sse_event({"event": "stream_end", "data": {"id": "final"}})

        # Execute cleanup function if provided
        if cleanup_func:
            try:
                if inspect.iscoroutinefunction(cleanup_func):  # Check if it's an async def function
                    await cleanup_func()
                elif inspect.isasyncgenfunction(cleanup_func) or inspect.isasyncgen(
                    cleanup_func):  # Check if it's an async def generator function or already an async generator
                    # If it's a function, call it to get the generator
                    gen_to_exhaust = cleanup_func() if inspect.isasyncgenfunction(cleanup_func) else cleanup_func
                    async for _ in gen_to_exhaust:
                        pass  # Exhaust the generator to ensure cleanup completes
                else:
                    # Synchronous function
                    cleanup_func()
            except Exception as e:
                # Log cleanup errors but don't propagate them to client
                error_info_cleanup = {
                    "event": "cleanup_error",
                    "data": {  # Ensure payload is under 'data' key
                        "message": str(e),
                        "traceback": traceback.format_exc()
                    }
                }
                # We can't yield here as the stream is already closing/closed.
                # Instead, log the error.
                # In a real app, use a proper logger.
                print(f"SSE cleanup error: {cls.format_sse_event(error_info_cleanup)}", flush=True)

format_sse_event(data) staticmethod

Format any data as a proper SSE event message.

Source code in toolboxv2/utils/system/types.py

@staticmethod
def format_sse_event(data: Any) -> str:
    """Format any data as a proper SSE event message."""
    # Already formatted as SSE
    if isinstance(data, str) and (data.startswith('data:') or data.startswith('event:')) and '\n\n' in data:
        return data

    # Handle bytes (binary data)
    if isinstance(data, bytes):
        try:
            # Try to decode as UTF-8 first
            decoded_data_str = data.decode('utf-8')
            # If decoding works, treat it as a string for further processing
            # This allows binary data that is valid UTF-8 JSON to be processed as JSON.
            data = decoded_data_str
        except UnicodeDecodeError:
            # Binary data that is not UTF-8, encode as base64
            b64_data = base64.b64encode(data).decode('utf-8')
            return f"event: binary\ndata: {b64_data}\n\n"

    # Convert non-string objects (that are not already bytes) to JSON string
    # If data was bytes and successfully decoded to UTF-8 string, it will be processed here.
    original_data_type_was_complex = False
    if not isinstance(data, str):
        original_data_type_was_complex = True
        try:
            data_str = json.dumps(data)
        except Exception:
            data_str = str(data)  # Fallback to string representation
    else:
        data_str = data  # data is already a string

    # Handle JSON data with special event formatting
    # data_str now holds the string representation (either original string or JSON string)
    if data_str.strip().startswith('{'):
        try:
            json_data = json.loads(data_str)
            if isinstance(json_data, dict) and 'event' in json_data:
                event_type = json_data['event']
                event_id = json_data.get('id', None)  # Use None to distinguish from empty string

                # Determine the actual data payload for the SSE 'data:' field
                # If 'data' key exists in json_data, use its content.
                # Otherwise, use the original data_str (which is the JSON of json_data).
                if 'data' in json_data:
                    payload_content = json_data['data']
                    # If payload_content is complex, re-serialize it to JSON string
                    if isinstance(payload_content, dict | list):
                        sse_data_field = json.dumps(payload_content)
                    else:  # Simple type (string, number, bool)
                        sse_data_field = str(payload_content)
                else:
                    # If original data was complex (e.g. dict) and became json_data,
                    # and no 'data' key in it, then use the full json_data as payload.
                    # If original data was a simple string that happened to be JSON parsable
                    # but without 'event' key, it would have been handled by "Regular JSON without event"
                    # or "Plain text" later.
                    # This path implies original data was a dict with 'event' key.
                    sse_data_field = data_str

                sse_lines = []
                if event_type:  # Should always be true here
                    sse_lines.append(f"event: {event_type}")
                if event_id is not None:  # Check for None, allow empty string id
                    sse_lines.append(f"id: {event_id}")

                # Handle multi-line data for the data field
                for line in sse_data_field.splitlines():
                    sse_lines.append(f"data: {line}")

                return "\n".join(sse_lines) + "\n\n"
            else:
                # Regular JSON without special 'event' key
                sse_lines = []
                for line in data_str.splitlines():
                    sse_lines.append(f"data: {line}")
                return "\n".join(sse_lines) + "\n\n"
        except json.JSONDecodeError:
            # Not valid JSON, treat as plain text
            sse_lines = []
            for line in data_str.splitlines():
                sse_lines.append(f"data: {line}")
            return "\n".join(sse_lines) + "\n\n"
    else:
        # Plain text
        sse_lines = []
        for line in data_str.splitlines():
            sse_lines.append(f"data: {line}")
        return "\n".join(sse_lines) + "\n\n"

wrap_sync_generator(generator) async classmethod

Convert a synchronous generator to an async generator.

Source code in toolboxv2/utils/system/types.py

@classmethod
async def wrap_sync_generator(cls, generator):
    """Convert a synchronous generator to an async generator."""
    for item in generator:
        yield item
        # Allow other tasks to run
        await asyncio.sleep(0)

Session

Class representing a session.

Source code in toolboxv2/utils/system/types.py

@dataclass
class Session:
    """Class representing a session."""
    SiID: str
    level: str
    spec: str
    user_name: str
    # Allow for additional fields
    extra_data: dict[str, Any] = field(default_factory=dict)

    @classmethod
    def from_dict(cls, data: dict[str, Any]) -> 'Session':
        """Create a Session instance from a dictionary with default values."""
        known_fields = {
            'SiID': data.get('SiID', '#0'),
            'level': data.get('level', -1),
            'spec': data.get('spec', 'app'),
            'user_name': data.get('user_name', 'anonymous'),
        }

        extra_data = {k: v for k, v in data.items() if k not in known_fields}
        return cls(**known_fields, extra_data=extra_data)

    def to_dict(self) -> dict[str, Any]:
        """Convert the Session object back to a dictionary."""
        result = {
            'SiID': self.SiID,
            'level': self.level,
            'spec': self.spec,
            'user_name': self.user_name,
        }

        # Add extra data
        result.update(self.extra_data)

        return result

    @property
    def valid(self):
        return int(self.level) > 0

from_dict(data) classmethod

Create a Session instance from a dictionary with default values.

Source code in toolboxv2/utils/system/types.py

@classmethod
def from_dict(cls, data: dict[str, Any]) -> 'Session':
    """Create a Session instance from a dictionary with default values."""
    known_fields = {
        'SiID': data.get('SiID', '#0'),
        'level': data.get('level', -1),
        'spec': data.get('spec', 'app'),
        'user_name': data.get('user_name', 'anonymous'),
    }

    extra_data = {k: v for k, v in data.items() if k not in known_fields}
    return cls(**known_fields, extra_data=extra_data)

to_dict()

Convert the Session object back to a dictionary.

Source code in toolboxv2/utils/system/types.py

def to_dict(self) -> dict[str, Any]:
    """Convert the Session object back to a dictionary."""
    result = {
        'SiID': self.SiID,
        'level': self.level,
        'spec': self.spec,
        'user_name': self.user_name,
    }

    # Add extra data
    result.update(self.extra_data)

    return result

parse_request_data(data)

Parse the incoming request data into a strongly typed structure.

Source code in toolboxv2/utils/system/types.py

def parse_request_data(data: dict[str, Any]) -> RequestData:
    """Parse the incoming request data into a strongly typed structure."""
    return RequestData.from_dict(data)

tbx

install_support

Complete TB Language Setup - Build executable - Setup file associations - Install VS Code extension - Install PyCharm plugin

TBSetup

Complete TB Language setup manager

Source code in toolboxv2/utils/tbx/install_support.py

class TBSetup:
    """Complete TB Language setup manager"""

    def __init__(self):
        self.root = Path(__file__).parent
        self.system = platform.system()

    def setup_all(self):
        """Run complete setup"""
        print("═" * 70)
        print("  TB Language - Complete Setup")
        print("═" * 70)
        print()

        success = True

        # Step 1: Build
        if not self.build_executable():
            print("❌ Build failed!")
            return False

        # Step 2: System integration
        if not self.setup_system_integration():
            print("⚠️  System integration failed (optional)")
            success = False

        # Step 3: VS Code extension
        if not self.setup_vscode():
            print("⚠️  VS Code extension setup failed (optional)")
            success = False

        # Step 4: PyCharm plugin
        if not self.setup_pycharm():
            print("⚠️  PyCharm plugin setup failed (optional)")
            success = False

        print()
        print("═" * 70)
        if success:
            print("  ✓ Setup Complete!")
        else:
            print("  ⚠️  Setup completed with warnings")
        print("═" * 70)
        print()
        print("Next steps:")
        print("  1. Restart PyCharm and VS Code (if open)")
        print("  2. Create a test file: test.tbx")
        print("  3. Run it: tb run test.tbx")
        print("  4. Or double-click test.tbx to run")
        print("  5. Open .tbx files in PyCharm/VS Code for syntax highlighting")
        print()

        return success

    def build_executable(self):
        """Step 1: Build TB Language"""
        print("Step 1/4: Building TB Language...")
        print("-" * 70)

        result = subprocess.run([
            sys.executable,
            str(self.root / "toolbox-exec" / "tb_lang_cli.py"),
            "build"
        ])

        if result.returncode != 0:
            return False

        print("✓ Build successful")
        print()
        return True

    def setup_system_integration(self):
        """Step 2: System integration"""
        print("Step 2/4: Setting up system integration...")
        print("-" * 70)

        result = subprocess.run([
            sys.executable,
            str(self.root / "toolbox-exec" / "tb_setup.py"),
            "install"
        ])

        print()
        return result.returncode == 0

    def setup_vscode(self):
        """Step 3: VS Code extension"""
        print("Step 3/4: Installing VS Code extension...")
        print("-" * 70)

        vscode_ext = self.root / "tb-lang-vscode"
        if not vscode_ext.exists():
            print("⚠️  VS Code extension directory not found")
            print()
            return False

        try:
            # Check if npm is available
            subprocess.run(["npm", "--version"],
                           capture_output=True, check=True)

            # Install dependencies
            print("  Installing npm dependencies...")
            subprocess.run(["npm", "install"],
                           cwd=vscode_ext,
                           capture_output=True,
                           check=True)

            # Compile TypeScript
            print("  Compiling TypeScript...")
            subprocess.run(["npm", "run", "compile"],
                           cwd=vscode_ext,
                           capture_output=True,
                           check=True)

            # Try to install to VS Code
            print("  Installing to VS Code...")
            result = subprocess.run([
                "code", "--install-extension", str(vscode_ext.resolve())
            ], capture_output=True)

            if result.returncode == 0:
                print("✓ VS Code extension installed")
                print()
                return True
            else:
                print("⚠️  Could not auto-install to VS Code")
                print(f"   Manual install: code --install-extension {vscode_ext.resolve()}")
                print()
                return False

        except FileNotFoundError as e:
            print(f"⚠️  Tool not found: {e}")
            print("   npm: https://nodejs.org/")
            print("   VS Code: https://code.visualstudio.com/")
            print()
            return False
        except subprocess.CalledProcessError as e:
            print(f"⚠️  Command failed: {e}")
            print()
            return False

    def setup_pycharm(self):
        """Step 4: PyCharm plugin"""
        print("Step 4/4: Installing PyCharm plugin...")
        print("-" * 70)

        pycharm_plugin = self.root / "tb-lang-pycharm"
        if not pycharm_plugin.exists():
            print("⚠️  PyCharm plugin directory not found")
            print("   Creating plugin structure...")
            if not self.create_pycharm_plugin():
                print()
                return False

        try:
            # Build plugin JAR
            print("  Building PyCharm plugin...")
            if not self.build_pycharm_plugin():
                print("⚠️  Plugin build failed")
                print()
                return False

            # Install to PyCharm
            print("  Installing to PyCharm...")
            if not self.install_pycharm_plugin():
                print("⚠️  Auto-install failed")
                print()
                return False

            print("✓ PyCharm plugin installed")
            print("  Please restart PyCharm to activate the plugin")
            print()
            return True

        except Exception as e:
            print(f"⚠️  Error: {e}")
            print()
            return False

    def create_pycharm_plugin(self):
        """Create PyCharm plugin structure"""
        plugin_dir = self.root / "tb-lang-pycharm"
        plugin_dir.mkdir(exist_ok=True)

        # Create directory structure
        (plugin_dir / "src" / "main" / "resources" / "fileTypes").mkdir(parents=True, exist_ok=True)
        (plugin_dir / "src" / "main" / "resources" / "META-INF").mkdir(parents=True, exist_ok=True)

        return True

    def build_pycharm_plugin(self):
        """Build PyCharm plugin JAR"""
        plugin_dir = self.root / "tb-lang-pycharm"
        build_script = plugin_dir / "build_plugin.py"

        if not build_script.exists():
            # Create build script
            build_script.write_text('''#!/usr/bin/env python3
import zipfile
from pathlib import Path

plugin_dir = Path(__file__).parent
output_jar = plugin_dir / "tb-language.jar"

with zipfile.ZipFile(output_jar, 'w', zipfile.ZIP_DEFLATED) as jar:
    # Add plugin.xml
    plugin_xml = plugin_dir / "src" / "main" / "resources" / "META-INF" / "plugin.xml"
    if plugin_xml.exists():
        jar.write(plugin_xml, "META-INF/plugin.xml")

    # Add file type definition
    file_type = plugin_dir / "src" / "main" / "resources" / "fileTypes" / "TB.xml"
    if file_type.exists():
        jar.write(file_type, "fileTypes/TB.xml")

print(f"✓ Plugin built: {output_jar}")
''')
            build_script.chmod(0o755)

        # Run build script
        result = subprocess.run([sys.executable, str(build_script)],
                                capture_output=True, text=True)

        if result.returncode == 0:
            print(f"  {result.stdout.strip()}")
            return True
        else:
            print(f"  Build error: {result.stderr}")
            return False

    def install_pycharm_plugin(self):
        """Install plugin to PyCharm"""
        plugin_jar = self.root / "tb-lang-pycharm" / "tb-language.jar"

        if not plugin_jar.exists():
            print("  Plugin JAR not found")
            return False

        # Find PyCharm config directory
        pycharm_dirs = self.find_pycharm_config_dirs()

        if not pycharm_dirs:
            print("  PyCharm installation not found")
            print(f"  Manual install: Copy {plugin_jar} to PyCharm plugins directory")
            return False

        # Install to all found PyCharm installations
        installed = False
        for config_dir in pycharm_dirs:
            plugins_dir = config_dir / "plugins"
            plugins_dir.mkdir(exist_ok=True)

            dest = plugins_dir / "tb-language.jar"
            shutil.copy(plugin_jar, dest)
            print(f"  ✓ Installed to: {dest}")
            installed = True

        return installed

    def find_pycharm_config_dirs(self):
        """Find PyCharm config directories"""
        config_dirs = []
        home = Path.home()

        if self.system == "Windows":
            # Windows: C:\Users\<user>\AppData\Roaming\JetBrains\PyCharm*
            base = home / "AppData" / "Roaming" / "JetBrains"
            if base.exists():
                config_dirs.extend(base.glob("PyCharm*"))

        elif self.system == "Linux":
            # Linux: ~/.config/JetBrains/PyCharm*
            base = home / ".config" / "JetBrains"
            if base.exists():
                config_dirs.extend(base.glob("PyCharm*"))

            # Also check old location
            old_base = home / ".PyCharm*"
            config_dirs.extend(home.glob(".PyCharm*"))

        elif self.system == "Darwin":
            # macOS: ~/Library/Application Support/JetBrains/PyCharm*
            base = home / "Library" / "Application Support" / "JetBrains"
            if base.exists():
                config_dirs.extend(base.glob("PyCharm*"))

        return [d for d in config_dirs if d.is_dir()]

build_executable()

Step 1: Build TB Language

Source code in toolboxv2/utils/tbx/install_support.py

def build_executable(self):
    """Step 1: Build TB Language"""
    print("Step 1/4: Building TB Language...")
    print("-" * 70)

    result = subprocess.run([
        sys.executable,
        str(self.root / "toolbox-exec" / "tb_lang_cli.py"),
        "build"
    ])

    if result.returncode != 0:
        return False

    print("✓ Build successful")
    print()
    return True

build_pycharm_plugin()

Build PyCharm plugin JAR

Source code in toolboxv2/utils/tbx/install_support.py

    def build_pycharm_plugin(self):
        """Build PyCharm plugin JAR"""
        plugin_dir = self.root / "tb-lang-pycharm"
        build_script = plugin_dir / "build_plugin.py"

        if not build_script.exists():
            # Create build script
            build_script.write_text('''#!/usr/bin/env python3
import zipfile
from pathlib import Path

plugin_dir = Path(__file__).parent
output_jar = plugin_dir / "tb-language.jar"

with zipfile.ZipFile(output_jar, 'w', zipfile.ZIP_DEFLATED) as jar:
    # Add plugin.xml
    plugin_xml = plugin_dir / "src" / "main" / "resources" / "META-INF" / "plugin.xml"
    if plugin_xml.exists():
        jar.write(plugin_xml, "META-INF/plugin.xml")

    # Add file type definition
    file_type = plugin_dir / "src" / "main" / "resources" / "fileTypes" / "TB.xml"
    if file_type.exists():
        jar.write(file_type, "fileTypes/TB.xml")

print(f"✓ Plugin built: {output_jar}")
''')
            build_script.chmod(0o755)

        # Run build script
        result = subprocess.run([sys.executable, str(build_script)],
                                capture_output=True, text=True)

        if result.returncode == 0:
            print(f"  {result.stdout.strip()}")
            return True
        else:
            print(f"  Build error: {result.stderr}")
            return False

create_pycharm_plugin()

Create PyCharm plugin structure

Source code in toolboxv2/utils/tbx/install_support.py

def create_pycharm_plugin(self):
    """Create PyCharm plugin structure"""
    plugin_dir = self.root / "tb-lang-pycharm"
    plugin_dir.mkdir(exist_ok=True)

    # Create directory structure
    (plugin_dir / "src" / "main" / "resources" / "fileTypes").mkdir(parents=True, exist_ok=True)
    (plugin_dir / "src" / "main" / "resources" / "META-INF").mkdir(parents=True, exist_ok=True)

    return True

find_pycharm_config_dirs()

Find PyCharm config directories

Source code in toolboxv2/utils/tbx/install_support.py

def find_pycharm_config_dirs(self):
    """Find PyCharm config directories"""
    config_dirs = []
    home = Path.home()

    if self.system == "Windows":
        # Windows: C:\Users\<user>\AppData\Roaming\JetBrains\PyCharm*
        base = home / "AppData" / "Roaming" / "JetBrains"
        if base.exists():
            config_dirs.extend(base.glob("PyCharm*"))

    elif self.system == "Linux":
        # Linux: ~/.config/JetBrains/PyCharm*
        base = home / ".config" / "JetBrains"
        if base.exists():
            config_dirs.extend(base.glob("PyCharm*"))

        # Also check old location
        old_base = home / ".PyCharm*"
        config_dirs.extend(home.glob(".PyCharm*"))

    elif self.system == "Darwin":
        # macOS: ~/Library/Application Support/JetBrains/PyCharm*
        base = home / "Library" / "Application Support" / "JetBrains"
        if base.exists():
            config_dirs.extend(base.glob("PyCharm*"))

    return [d for d in config_dirs if d.is_dir()]

install_pycharm_plugin()

Install plugin to PyCharm

Source code in toolboxv2/utils/tbx/install_support.py

def install_pycharm_plugin(self):
    """Install plugin to PyCharm"""
    plugin_jar = self.root / "tb-lang-pycharm" / "tb-language.jar"

    if not plugin_jar.exists():
        print("  Plugin JAR not found")
        return False

    # Find PyCharm config directory
    pycharm_dirs = self.find_pycharm_config_dirs()

    if not pycharm_dirs:
        print("  PyCharm installation not found")
        print(f"  Manual install: Copy {plugin_jar} to PyCharm plugins directory")
        return False

    # Install to all found PyCharm installations
    installed = False
    for config_dir in pycharm_dirs:
        plugins_dir = config_dir / "plugins"
        plugins_dir.mkdir(exist_ok=True)

        dest = plugins_dir / "tb-language.jar"
        shutil.copy(plugin_jar, dest)
        print(f"  ✓ Installed to: {dest}")
        installed = True

    return installed

setup_all()

Run complete setup

Source code in toolboxv2/utils/tbx/install_support.py

def setup_all(self):
    """Run complete setup"""
    print("═" * 70)
    print("  TB Language - Complete Setup")
    print("═" * 70)
    print()

    success = True

    # Step 1: Build
    if not self.build_executable():
        print("❌ Build failed!")
        return False

    # Step 2: System integration
    if not self.setup_system_integration():
        print("⚠️  System integration failed (optional)")
        success = False

    # Step 3: VS Code extension
    if not self.setup_vscode():
        print("⚠️  VS Code extension setup failed (optional)")
        success = False

    # Step 4: PyCharm plugin
    if not self.setup_pycharm():
        print("⚠️  PyCharm plugin setup failed (optional)")
        success = False

    print()
    print("═" * 70)
    if success:
        print("  ✓ Setup Complete!")
    else:
        print("  ⚠️  Setup completed with warnings")
    print("═" * 70)
    print()
    print("Next steps:")
    print("  1. Restart PyCharm and VS Code (if open)")
    print("  2. Create a test file: test.tbx")
    print("  3. Run it: tb run test.tbx")
    print("  4. Or double-click test.tbx to run")
    print("  5. Open .tbx files in PyCharm/VS Code for syntax highlighting")
    print()

    return success

setup_pycharm()

Step 4: PyCharm plugin

Source code in toolboxv2/utils/tbx/install_support.py

def setup_pycharm(self):
    """Step 4: PyCharm plugin"""
    print("Step 4/4: Installing PyCharm plugin...")
    print("-" * 70)

    pycharm_plugin = self.root / "tb-lang-pycharm"
    if not pycharm_plugin.exists():
        print("⚠️  PyCharm plugin directory not found")
        print("   Creating plugin structure...")
        if not self.create_pycharm_plugin():
            print()
            return False

    try:
        # Build plugin JAR
        print("  Building PyCharm plugin...")
        if not self.build_pycharm_plugin():
            print("⚠️  Plugin build failed")
            print()
            return False

        # Install to PyCharm
        print("  Installing to PyCharm...")
        if not self.install_pycharm_plugin():
            print("⚠️  Auto-install failed")
            print()
            return False

        print("✓ PyCharm plugin installed")
        print("  Please restart PyCharm to activate the plugin")
        print()
        return True

    except Exception as e:
        print(f"⚠️  Error: {e}")
        print()
        return False

setup_system_integration()

Step 2: System integration

Source code in toolboxv2/utils/tbx/install_support.py

def setup_system_integration(self):
    """Step 2: System integration"""
    print("Step 2/4: Setting up system integration...")
    print("-" * 70)

    result = subprocess.run([
        sys.executable,
        str(self.root / "toolbox-exec" / "tb_setup.py"),
        "install"
    ])

    print()
    return result.returncode == 0

setup_vscode()

Step 3: VS Code extension

Source code in toolboxv2/utils/tbx/install_support.py

def setup_vscode(self):
    """Step 3: VS Code extension"""
    print("Step 3/4: Installing VS Code extension...")
    print("-" * 70)

    vscode_ext = self.root / "tb-lang-vscode"
    if not vscode_ext.exists():
        print("⚠️  VS Code extension directory not found")
        print()
        return False

    try:
        # Check if npm is available
        subprocess.run(["npm", "--version"],
                       capture_output=True, check=True)

        # Install dependencies
        print("  Installing npm dependencies...")
        subprocess.run(["npm", "install"],
                       cwd=vscode_ext,
                       capture_output=True,
                       check=True)

        # Compile TypeScript
        print("  Compiling TypeScript...")
        subprocess.run(["npm", "run", "compile"],
                       cwd=vscode_ext,
                       capture_output=True,
                       check=True)

        # Try to install to VS Code
        print("  Installing to VS Code...")
        result = subprocess.run([
            "code", "--install-extension", str(vscode_ext.resolve())
        ], capture_output=True)

        if result.returncode == 0:
            print("✓ VS Code extension installed")
            print()
            return True
        else:
            print("⚠️  Could not auto-install to VS Code")
            print(f"   Manual install: code --install-extension {vscode_ext.resolve()}")
            print()
            return False

    except FileNotFoundError as e:
        print(f"⚠️  Tool not found: {e}")
        print("   npm: https://nodejs.org/")
        print("   VS Code: https://code.visualstudio.com/")
        print()
        return False
    except subprocess.CalledProcessError as e:
        print(f"⚠️  Command failed: {e}")
        print()
        return False

main()

Main entry point

Source code in toolboxv2/utils/tbx/install_support.py

def main():
    """Main entry point"""
    import argparse

    parser = argparse.ArgumentParser(
        description="TB Language Complete Setup"
    )
    parser.add_argument('--skip-build', action='store_true',
                        help='Skip building the executable')
    parser.add_argument('--skip-system', action='store_true',
                        help='Skip system integration')
    parser.add_argument('--skip-vscode', action='store_true',
                        help='Skip VS Code extension')
    parser.add_argument('--skip-pycharm', action='store_true',
                        help='Skip PyCharm plugin')
    parser.add_argument('--pycharm-only', action='store_true',
                        help='Only setup PyCharm plugin')

    args = parser.parse_args()

    setup = TBSetup()

    if args.pycharm_only:
        success = setup.setup_pycharm()
    else:
        # Full setup with skip options
        success = True

        if not args.skip_build:
            success = setup.build_executable() and success

        if not args.skip_system:
            setup.setup_system_integration()

        if not args.skip_vscode:
            setup.setup_vscode()

        if not args.skip_pycharm:
            setup.setup_pycharm()

    sys.exit(0 if success else 1)

setup

TB Language Setup Utility - File association (.tbx files) - Icon registration - Desktop integration

TBxSetup

Setup utility for TB Language file associations and icons

Source code in toolboxv2/utils/tbx/setup.py

class TBxSetup:
    """Setup utility for TB Language file associations and icons"""

    def __init__(self):
        self.system = platform.system()
        self.tb_root = self.get_tb_root()
        self.icon_path = Path(os.getenv("FAVI", ".ico"))
        self.executable = self.get_executable()

    def get_tb_root(self) -> Path:
        """Get toolbox root directory"""
        try:
            from toolboxv2 import tb_root_dir
            return Path(tb_root_dir)
        except ImportError:
            return Path(__file__).parent.parent

    def get_executable(self) -> Path:
        """Get TB executable path"""
        if self.system == "Windows":
            exe = self.tb_root / "bin" / "tb.exe"
        else:
            exe = self.tb_root / "bin" / "tb"

        if not exe.exists():
            # Try target/release
            if self.system == "Windows":
                exe = self.tb_root / "tb-exc" / "target" / "release" / "tb.exe"
            else:
                exe = self.tb_root / "tb-exc" / "target" / "release" / "tb"

        return exe

    def setup_all(self):
        """Run complete setup"""
        print("╔════════════════════════════════════════════════════════════════╗")
        print("║         TB Language - System Integration Setup                 ║")
        print("╚════════════════════════════════════════════════════════════════╝")
        print()

        # Check prerequisites
        if not self.executable.exists():
            print("❌ TB executable not found!")
            print(f"   Expected at: {self.executable}")
            print("   Run 'tb x build' first!")
            return False

        print(f"✓ TB executable found: {self.executable}")
        print()

        # Setup icon
        if not self.setup_icon():
            print("⚠️  Icon setup failed (continuing anyway)")

        # Setup file association
        if self.system == "Windows":
            success = self.setup_windows()
        elif self.system == "Linux":
            success = self.setup_linux()
        elif self.system == "Darwin":
            success = self.setup_macos()
        else:
            print(f"❌ Unsupported system: {self.system}")
            return False

        if success:
            print()
            print("╔════════════════════════════════════════════════════════════════╗")
            print("║                    ✓ Setup Complete!                           ║")
            print("╠════════════════════════════════════════════════════════════════╣")
            print("║  .tbx files are now associated with TB Language                ║")
            print("║  Double-click any .tbx file to run it!                         ║")
            print("╚════════════════════════════════════════════════════════════════╝")

        return success

    def setup_icon(self) -> bool:
        """Setup icon file"""
        print("📦 Setting up icon...")

        icon_dir = self.tb_root / "resources"
        icon_dir.mkdir(exist_ok=True)

        # Check if icon exists
        if self.icon_path.exists():
            print(f"   ✓ Icon already exists: {self.icon_path}")
            return True

        # Create placeholder icon info
        print(f"   ⚠️  Icon not found at: {self.icon_path}")
        print(f"   📝 Creating placeholder...")

        # Try to create a simple icon reference
        # User needs to provide actual tb_icon.ico file
        placeholder = icon_dir / "README_ICON.txt"
        placeholder.write_text("""
TB Language Icon
================

Place your icon files here:
- tb_icon.ico   (Windows)
- tb_icon.png   (Linux)
- tb_icon.icns  (macOS)

Recommended size: 256x256 px

You can use the ToolBox V2 logo/icon.
        """)

        print(f"   ℹ️  Place icon file at: {self.icon_path}")
        return False

    def setup_windows(self) -> bool:
        """Setup file association on Windows"""
        print("🪟 Setting up Windows file association...")

        try:
            import winreg

            # Create .tbx extension key
            print("   Creating registry entries...")

            # HKEY_CURRENT_USER\Software\Classes\.tbx
            with winreg.CreateKey(winreg.HKEY_CURRENT_USER, r"Software\Classes\.tbx") as key:
                winreg.SetValue(key, "", winreg.REG_SZ, "TBLanguageFile")
                print("   ✓ Registered .tbx extension")

            # HKEY_CURRENT_USER\Software\Classes\TBLanguageFile
            with winreg.CreateKey(winreg.HKEY_CURRENT_USER, r"Software\Classes\TBLanguageFile") as key:
                winreg.SetValue(key, "", winreg.REG_SZ, "TB Language Program")

                # Set icon
                if self.icon_path.exists():
                    icon_key = winreg.CreateKey(key, "DefaultIcon")
                    winreg.SetValue(icon_key, "", winreg.REG_SZ, str(self.icon_path))
                    print(f"   ✓ Set icon: {self.icon_path}")

                # Set open command
                command_key = winreg.CreateKey(key, r"shell\open\command")
                cmd = f'"{self.executable}" run "%1"'
                winreg.SetValue(command_key, "", winreg.REG_SZ, cmd)
                print(f"   ✓ Set open command: {cmd}")

                # Add "Run in Terminal" context menu
                terminal_key = winreg.CreateKey(key, r"shell\run_terminal\command")
                terminal_cmd = f'cmd /k "{self.executable}" run "%1" && pause'
                winreg.SetValue(terminal_key, "", winreg.REG_SZ, terminal_cmd)
                winreg.SetValue(winreg.CreateKey(key, r"shell\run_terminal"), "", winreg.REG_SZ, "Run in Terminal")
                print(f"   ✓ Added 'Run in Terminal' context menu")

                # Add "Edit" context menu
                edit_key = winreg.CreateKey(key, r"shell\edit\command")
                winreg.SetValue(edit_key, "", winreg.REG_SZ, 'notepad "%1"')
                winreg.SetValue(winreg.CreateKey(key, r"shell\edit"), "", winreg.REG_SZ, "Edit")
                print(f"   ✓ Added 'Edit' context menu")

            # Refresh shell
            print("   Refreshing Explorer...")
            try:
                import ctypes
                ctypes.windll.shell32.SHChangeNotify(0x08000000, 0x0000, None, None)
            except:
                print("   ⚠️  Could not refresh Explorer (restart may be needed)")

            print("   ✓ Windows setup complete!")
            return True

        except ImportError:
            print("   ❌ winreg module not available")
            return False
        except Exception as e:
            print(f"   ❌ Error: {e}")
            return False

    def setup_linux(self) -> bool:
        """Setup file association on Linux"""
        print("🐧 Setting up Linux file association...")

        try:
            # Create .desktop file
            desktop_dir = Path.home() / ".local" / "share" / "applications"
            desktop_dir.mkdir(parents=True, exist_ok=True)

            desktop_file = desktop_dir / "tb-language.desktop"

            icon_path = self.icon_path.with_suffix('.png')
            if not icon_path.exists():
                icon_path = "text-x-script"  # Fallback icon

            desktop_content = f"""[Desktop Entry]
Version=1.0
Type=Application
Name=TB Language
Comment=Execute TB Language programs
Exec={self.executable} run %f
Icon={icon_path}
Terminal=false
MimeType=text/x-tb;application/x-tb;
Categories=Development;
"""

            desktop_file.write_text(desktop_content)
            desktop_file.chmod(0o755)
            print(f"   ✓ Created desktop entry: {desktop_file}")

            # Create MIME type
            mime_dir = Path.home() / ".local" / "share" / "mime" / "packages"
            mime_dir.mkdir(parents=True, exist_ok=True)

            mime_file = mime_dir / "tb-language.xml"
            mime_content = """<?xml version="1.0" encoding="UTF-8"?>
<mime-info xmlns="http://www.freedesktop.org/standards/shared-mime-info">
    <mime-type type="text/x-tb">
        <comment>TB Language Program</comment>
        <glob pattern="*.tbx"/>
        <sub-class-of type="text/plain"/>
    </mime-type>
</mime-info>
"""

            mime_file.write_text(mime_content)
            print(f"   ✓ Created MIME type: {mime_file}")

            # Update MIME database
            print("   Updating MIME database...")
            try:
                subprocess.run(["update-mime-database",
                                str(Path.home() / ".local" / "share" / "mime")],
                               check=True, capture_output=True)
                print("   ✓ MIME database updated")
            except:
                print("   ⚠️  Could not update MIME database automatically")
                print("   Run: update-mime-database ~/.local/share/mime")

            # Update desktop database
            print("   Updating desktop database...")
            try:
                subprocess.run(["update-desktop-database", str(desktop_dir)],
                               check=True, capture_output=True)
                print("   ✓ Desktop database updated")
            except:
                print("   ⚠️  Could not update desktop database automatically")

            # Set default application
            try:
                subprocess.run([
                    "xdg-mime", "default", "tb-language.desktop", "text/x-tb"
                ], check=True, capture_output=True)
                print("   ✓ Set as default application for .tbx files")
            except:
                print("   ⚠️  Could not set as default application")

            print("   ✓ Linux setup complete!")
            return True

        except Exception as e:
            print(f"   ❌ Error: {e}")
            return False

    def setup_macos(self) -> bool:
        """Setup file association on macOS"""
        print("🍎 Setting up macOS file association...")

        try:
            # Create Info.plist for file association
            app_dir = self.tb_root / "TB Language.app"
            contents_dir = app_dir / "Contents"
            macos_dir = contents_dir / "MacOS"
            resources_dir = contents_dir / "Resources"

            # Create directories
            macos_dir.mkdir(parents=True, exist_ok=True)
            resources_dir.mkdir(parents=True, exist_ok=True)

            # Copy executable
            app_executable = macos_dir / "tb"
            if not app_executable.exists():
                shutil.copy(self.executable, app_executable)
                app_executable.chmod(0o755)

            # Create launcher script
            launcher = macos_dir / "TB Language"
            launcher.write_text(f"""#!/bin/bash
if [ "$#" -gt 0 ]; then
    "{app_executable}" run "$@"
else
    "{app_executable}" repl
fi
""")
            launcher.chmod(0o755)

            # Create Info.plist
            plist_file = contents_dir / "Info.plist"
            plist_content = f"""<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
    <key>CFBundleExecutable</key>
    <string>TB Language</string>
    <key>CFBundleIconFile</key>
    <string>tb_icon</string>
    <key>CFBundleIdentifier</key>
    <string>dev.tblang.tb</string>
    <key>CFBundleName</key>
    <string>TB Language</string>
    <key>CFBundlePackageType</key>
    <string>APPL</string>
    <key>CFBundleShortVersionString</key>
    <string>1.0.0</string>
    <key>CFBundleVersion</key>
    <string>1.0.0</string>
    <key>CFBundleDocumentTypes</key>
    <array>
        <dict>
            <key>CFBundleTypeExtensions</key>
            <array>
                <string>tbx</string>
            </array>
            <key>CFBundleTypeIconFile</key>
            <string>tb_icon</string>
            <key>CFBundleTypeName</key>
            <string>TB Language Program</string>
            <key>CFBundleTypeRole</key>
            <string>Editor</string>
            <key>LSHandlerRank</key>
            <string>Owner</string>
        </dict>
    </array>
</dict>
</plist>
"""
            plist_file.write_text(plist_content)
            print(f"   ✓ Created app bundle: {app_dir}")

            # Copy icon if exists
            icon_src = self.icon_path.with_suffix('.icns')
            if icon_src.exists():
                shutil.copy(icon_src, resources_dir / "tb_icon.icns")
                print(f"   ✓ Copied icon")

            # Register with Launch Services
            print("   Registering with Launch Services...")
            try:
                subprocess.run([
                    "/System/Library/Frameworks/CoreServices.framework/Frameworks/LaunchServices.framework/Support/lsregister",
                    "-f", str(app_dir)
                ], check=True, capture_output=True)
                print("   ✓ Registered with Launch Services")
            except:
                print("   ⚠️  Could not register automatically")
                print(f"   Run: open '{app_dir}'")

            print("   ✓ macOS setup complete!")
            return True

        except Exception as e:
            print(f"   ❌ Error: {e}")
            return False

    def uninstall(self):
        """Remove file associations"""
        print("🗑️  Uninstalling file associations...")

        if self.system == "Windows":
            try:
                import winreg
                winreg.DeleteKey(winreg.HKEY_CURRENT_USER, r"Software\Classes\.tbx")
                winreg.DeleteKey(winreg.HKEY_CURRENT_USER, r"Software\Classes\TBLanguageFile")
                print("   ✓ Windows registry cleaned")
            except:
                print("   ⚠️  Could not clean registry")

        elif self.system == "Linux":
            desktop_file = Path.home() / ".local" / "share" / "applications" / "tb-language.desktop"
            mime_file = Path.home() / ".local" / "share" / "mime" / "packages" / "tb-language.xml"

            if desktop_file.exists():
                desktop_file.unlink()
                print("   ✓ Removed desktop entry")

            if mime_file.exists():
                mime_file.unlink()
                print("   ✓ Removed MIME type")

        elif self.system == "Darwin":
            app_dir = self.tb_root / "TB Language.app"
            if app_dir.exists():
                shutil.rmtree(app_dir)
                print("   ✓ Removed app bundle")

        print("   ✓ Uninstall complete!")

get_executable()

Get TB executable path

Source code in toolboxv2/utils/tbx/setup.py

def get_executable(self) -> Path:
    """Get TB executable path"""
    if self.system == "Windows":
        exe = self.tb_root / "bin" / "tb.exe"
    else:
        exe = self.tb_root / "bin" / "tb"

    if not exe.exists():
        # Try target/release
        if self.system == "Windows":
            exe = self.tb_root / "tb-exc" / "target" / "release" / "tb.exe"
        else:
            exe = self.tb_root / "tb-exc" / "target" / "release" / "tb"

    return exe

get_tb_root()

Get toolbox root directory

Source code in toolboxv2/utils/tbx/setup.py

def get_tb_root(self) -> Path:
    """Get toolbox root directory"""
    try:
        from toolboxv2 import tb_root_dir
        return Path(tb_root_dir)
    except ImportError:
        return Path(__file__).parent.parent

setup_all()

Run complete setup

Source code in toolboxv2/utils/tbx/setup.py

def setup_all(self):
    """Run complete setup"""
    print("╔════════════════════════════════════════════════════════════════╗")
    print("║         TB Language - System Integration Setup                 ║")
    print("╚════════════════════════════════════════════════════════════════╝")
    print()

    # Check prerequisites
    if not self.executable.exists():
        print("❌ TB executable not found!")
        print(f"   Expected at: {self.executable}")
        print("   Run 'tb x build' first!")
        return False

    print(f"✓ TB executable found: {self.executable}")
    print()

    # Setup icon
    if not self.setup_icon():
        print("⚠️  Icon setup failed (continuing anyway)")

    # Setup file association
    if self.system == "Windows":
        success = self.setup_windows()
    elif self.system == "Linux":
        success = self.setup_linux()
    elif self.system == "Darwin":
        success = self.setup_macos()
    else:
        print(f"❌ Unsupported system: {self.system}")
        return False

    if success:
        print()
        print("╔════════════════════════════════════════════════════════════════╗")
        print("║                    ✓ Setup Complete!                           ║")
        print("╠════════════════════════════════════════════════════════════════╣")
        print("║  .tbx files are now associated with TB Language                ║")
        print("║  Double-click any .tbx file to run it!                         ║")
        print("╚════════════════════════════════════════════════════════════════╝")

    return success

setup_icon()

Setup icon file

Source code in toolboxv2/utils/tbx/setup.py

    def setup_icon(self) -> bool:
        """Setup icon file"""
        print("📦 Setting up icon...")

        icon_dir = self.tb_root / "resources"
        icon_dir.mkdir(exist_ok=True)

        # Check if icon exists
        if self.icon_path.exists():
            print(f"   ✓ Icon already exists: {self.icon_path}")
            return True

        # Create placeholder icon info
        print(f"   ⚠️  Icon not found at: {self.icon_path}")
        print(f"   📝 Creating placeholder...")

        # Try to create a simple icon reference
        # User needs to provide actual tb_icon.ico file
        placeholder = icon_dir / "README_ICON.txt"
        placeholder.write_text("""
TB Language Icon
================

Place your icon files here:
- tb_icon.ico   (Windows)
- tb_icon.png   (Linux)
- tb_icon.icns  (macOS)

Recommended size: 256x256 px

You can use the ToolBox V2 logo/icon.
        """)

        print(f"   ℹ️  Place icon file at: {self.icon_path}")
        return False

setup_linux()

Setup file association on Linux

Source code in toolboxv2/utils/tbx/setup.py

    def setup_linux(self) -> bool:
        """Setup file association on Linux"""
        print("🐧 Setting up Linux file association...")

        try:
            # Create .desktop file
            desktop_dir = Path.home() / ".local" / "share" / "applications"
            desktop_dir.mkdir(parents=True, exist_ok=True)

            desktop_file = desktop_dir / "tb-language.desktop"

            icon_path = self.icon_path.with_suffix('.png')
            if not icon_path.exists():
                icon_path = "text-x-script"  # Fallback icon

            desktop_content = f"""[Desktop Entry]
Version=1.0
Type=Application
Name=TB Language
Comment=Execute TB Language programs
Exec={self.executable} run %f
Icon={icon_path}
Terminal=false
MimeType=text/x-tb;application/x-tb;
Categories=Development;
"""

            desktop_file.write_text(desktop_content)
            desktop_file.chmod(0o755)
            print(f"   ✓ Created desktop entry: {desktop_file}")

            # Create MIME type
            mime_dir = Path.home() / ".local" / "share" / "mime" / "packages"
            mime_dir.mkdir(parents=True, exist_ok=True)

            mime_file = mime_dir / "tb-language.xml"
            mime_content = """<?xml version="1.0" encoding="UTF-8"?>
<mime-info xmlns="http://www.freedesktop.org/standards/shared-mime-info">
    <mime-type type="text/x-tb">
        <comment>TB Language Program</comment>
        <glob pattern="*.tbx"/>
        <sub-class-of type="text/plain"/>
    </mime-type>
</mime-info>
"""

            mime_file.write_text(mime_content)
            print(f"   ✓ Created MIME type: {mime_file}")

            # Update MIME database
            print("   Updating MIME database...")
            try:
                subprocess.run(["update-mime-database",
                                str(Path.home() / ".local" / "share" / "mime")],
                               check=True, capture_output=True)
                print("   ✓ MIME database updated")
            except:
                print("   ⚠️  Could not update MIME database automatically")
                print("   Run: update-mime-database ~/.local/share/mime")

            # Update desktop database
            print("   Updating desktop database...")
            try:
                subprocess.run(["update-desktop-database", str(desktop_dir)],
                               check=True, capture_output=True)
                print("   ✓ Desktop database updated")
            except:
                print("   ⚠️  Could not update desktop database automatically")

            # Set default application
            try:
                subprocess.run([
                    "xdg-mime", "default", "tb-language.desktop", "text/x-tb"
                ], check=True, capture_output=True)
                print("   ✓ Set as default application for .tbx files")
            except:
                print("   ⚠️  Could not set as default application")

            print("   ✓ Linux setup complete!")
            return True

        except Exception as e:
            print(f"   ❌ Error: {e}")
            return False

setup_macos()

Setup file association on macOS

Source code in toolboxv2/utils/tbx/setup.py

    def setup_macos(self) -> bool:
        """Setup file association on macOS"""
        print("🍎 Setting up macOS file association...")

        try:
            # Create Info.plist for file association
            app_dir = self.tb_root / "TB Language.app"
            contents_dir = app_dir / "Contents"
            macos_dir = contents_dir / "MacOS"
            resources_dir = contents_dir / "Resources"

            # Create directories
            macos_dir.mkdir(parents=True, exist_ok=True)
            resources_dir.mkdir(parents=True, exist_ok=True)

            # Copy executable
            app_executable = macos_dir / "tb"
            if not app_executable.exists():
                shutil.copy(self.executable, app_executable)
                app_executable.chmod(0o755)

            # Create launcher script
            launcher = macos_dir / "TB Language"
            launcher.write_text(f"""#!/bin/bash
if [ "$#" -gt 0 ]; then
    "{app_executable}" run "$@"
else
    "{app_executable}" repl
fi
""")
            launcher.chmod(0o755)

            # Create Info.plist
            plist_file = contents_dir / "Info.plist"
            plist_content = f"""<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
    <key>CFBundleExecutable</key>
    <string>TB Language</string>
    <key>CFBundleIconFile</key>
    <string>tb_icon</string>
    <key>CFBundleIdentifier</key>
    <string>dev.tblang.tb</string>
    <key>CFBundleName</key>
    <string>TB Language</string>
    <key>CFBundlePackageType</key>
    <string>APPL</string>
    <key>CFBundleShortVersionString</key>
    <string>1.0.0</string>
    <key>CFBundleVersion</key>
    <string>1.0.0</string>
    <key>CFBundleDocumentTypes</key>
    <array>
        <dict>
            <key>CFBundleTypeExtensions</key>
            <array>
                <string>tbx</string>
            </array>
            <key>CFBundleTypeIconFile</key>
            <string>tb_icon</string>
            <key>CFBundleTypeName</key>
            <string>TB Language Program</string>
            <key>CFBundleTypeRole</key>
            <string>Editor</string>
            <key>LSHandlerRank</key>
            <string>Owner</string>
        </dict>
    </array>
</dict>
</plist>
"""
            plist_file.write_text(plist_content)
            print(f"   ✓ Created app bundle: {app_dir}")

            # Copy icon if exists
            icon_src = self.icon_path.with_suffix('.icns')
            if icon_src.exists():
                shutil.copy(icon_src, resources_dir / "tb_icon.icns")
                print(f"   ✓ Copied icon")

            # Register with Launch Services
            print("   Registering with Launch Services...")
            try:
                subprocess.run([
                    "/System/Library/Frameworks/CoreServices.framework/Frameworks/LaunchServices.framework/Support/lsregister",
                    "-f", str(app_dir)
                ], check=True, capture_output=True)
                print("   ✓ Registered with Launch Services")
            except:
                print("   ⚠️  Could not register automatically")
                print(f"   Run: open '{app_dir}'")

            print("   ✓ macOS setup complete!")
            return True

        except Exception as e:
            print(f"   ❌ Error: {e}")
            return False

setup_windows()

Setup file association on Windows

Source code in toolboxv2/utils/tbx/setup.py

def setup_windows(self) -> bool:
    """Setup file association on Windows"""
    print("🪟 Setting up Windows file association...")

    try:
        import winreg

        # Create .tbx extension key
        print("   Creating registry entries...")

        # HKEY_CURRENT_USER\Software\Classes\.tbx
        with winreg.CreateKey(winreg.HKEY_CURRENT_USER, r"Software\Classes\.tbx") as key:
            winreg.SetValue(key, "", winreg.REG_SZ, "TBLanguageFile")
            print("   ✓ Registered .tbx extension")

        # HKEY_CURRENT_USER\Software\Classes\TBLanguageFile
        with winreg.CreateKey(winreg.HKEY_CURRENT_USER, r"Software\Classes\TBLanguageFile") as key:
            winreg.SetValue(key, "", winreg.REG_SZ, "TB Language Program")

            # Set icon
            if self.icon_path.exists():
                icon_key = winreg.CreateKey(key, "DefaultIcon")
                winreg.SetValue(icon_key, "", winreg.REG_SZ, str(self.icon_path))
                print(f"   ✓ Set icon: {self.icon_path}")

            # Set open command
            command_key = winreg.CreateKey(key, r"shell\open\command")
            cmd = f'"{self.executable}" run "%1"'
            winreg.SetValue(command_key, "", winreg.REG_SZ, cmd)
            print(f"   ✓ Set open command: {cmd}")

            # Add "Run in Terminal" context menu
            terminal_key = winreg.CreateKey(key, r"shell\run_terminal\command")
            terminal_cmd = f'cmd /k "{self.executable}" run "%1" && pause'
            winreg.SetValue(terminal_key, "", winreg.REG_SZ, terminal_cmd)
            winreg.SetValue(winreg.CreateKey(key, r"shell\run_terminal"), "", winreg.REG_SZ, "Run in Terminal")
            print(f"   ✓ Added 'Run in Terminal' context menu")

            # Add "Edit" context menu
            edit_key = winreg.CreateKey(key, r"shell\edit\command")
            winreg.SetValue(edit_key, "", winreg.REG_SZ, 'notepad "%1"')
            winreg.SetValue(winreg.CreateKey(key, r"shell\edit"), "", winreg.REG_SZ, "Edit")
            print(f"   ✓ Added 'Edit' context menu")

        # Refresh shell
        print("   Refreshing Explorer...")
        try:
            import ctypes
            ctypes.windll.shell32.SHChangeNotify(0x08000000, 0x0000, None, None)
        except:
            print("   ⚠️  Could not refresh Explorer (restart may be needed)")

        print("   ✓ Windows setup complete!")
        return True

    except ImportError:
        print("   ❌ winreg module not available")
        return False
    except Exception as e:
        print(f"   ❌ Error: {e}")
        return False

uninstall()

Remove file associations

Source code in toolboxv2/utils/tbx/setup.py

def uninstall(self):
    """Remove file associations"""
    print("🗑️  Uninstalling file associations...")

    if self.system == "Windows":
        try:
            import winreg
            winreg.DeleteKey(winreg.HKEY_CURRENT_USER, r"Software\Classes\.tbx")
            winreg.DeleteKey(winreg.HKEY_CURRENT_USER, r"Software\Classes\TBLanguageFile")
            print("   ✓ Windows registry cleaned")
        except:
            print("   ⚠️  Could not clean registry")

    elif self.system == "Linux":
        desktop_file = Path.home() / ".local" / "share" / "applications" / "tb-language.desktop"
        mime_file = Path.home() / ".local" / "share" / "mime" / "packages" / "tb-language.xml"

        if desktop_file.exists():
            desktop_file.unlink()
            print("   ✓ Removed desktop entry")

        if mime_file.exists():
            mime_file.unlink()
            print("   ✓ Removed MIME type")

    elif self.system == "Darwin":
        app_dir = self.tb_root / "TB Language.app"
        if app_dir.exists():
            shutil.rmtree(app_dir)
            print("   ✓ Removed app bundle")

    print("   ✓ Uninstall complete!")

main()

Main entry point

Source code in toolboxv2/utils/tbx/setup.py

def main():
    """Main entry point"""
    import argparse

    parser = argparse.ArgumentParser(
        description="TB Language System Integration Setup"
    )
    parser.add_argument('action', choices=['install', 'uninstall'],
                        help='Action to perform')

    args = parser.parse_args()

    setup = TBxSetup()

    if args.action == 'install':
        success = setup.setup_all()
        sys.exit(0 if success else 1)
    elif args.action == 'uninstall':
        setup.uninstall()
        sys.exit(0)

test

test_tb_lang

TB Language Comprehensive Test Suite Tests all features of the TB language implementation.

Usage

python test_tb_lang.py python test_tb_lang.py --verbose python test_tb_lang.py --filter "test_arithmetic"

assert_contains(code, substring, mode='jit')

Assert that output contains substring.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

def assert_contains(code: str, substring: str, mode: str = "jit"):
    """Assert that output contains substring."""
    success, stdout, stderr = run_tb(code, mode)

    if not success:
        raise AssertionError(f"Execution failed:\n{stderr}")

    if substring not in stdout:
        raise AssertionError(
            f"Output does not contain '{substring}':\n{stdout}"
        )

assert_output(code, expected, mode='jit')

Assert that TB code produces expected output.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

def assert_output(code: str, expected: str, mode: str = "jit"):
    """Assert that TB code produces expected output."""

    success, stdout, stderr = run_tb(code, mode)
    if VERBOSE and not success:
        print(f"code: {code}")
        print()
        print(f"stdout: {stdout}")
    if not success:
        raise AssertionError(f"Execution failed:\n{stderr}")

    actual = stdout.strip()
    expected = expected.strip()

    if actual != expected:
        raise AssertionError(
            f"Output mismatch:\n"
            f"Expected: {repr(expected)}\n"
            f"Got:      {repr(actual)}"
        )

assert_success(code, mode='jit')

Assert that TB code runs without error.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

def assert_success(code: str, mode: str = "jit"):
    """Assert that TB code runs without error."""
    success, stdout, stderr = run_tb(code, mode)
    if VERBOSE and not success:
        print(f"code: {code}")
        print()
    if VERBOSE:
        print(f"stdout: {stdout}")
        print(f"stderr: {stderr}")

    if not success:
        raise AssertionError(f"Execution failed:\n{stderr}")

find_tb_binary()

Find TB binary, trying multiple paths with system-specific extensions.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

def find_tb_binary() -> str:
    """Find TB binary, trying multiple paths with system-specific extensions."""
    from toolboxv2 import tb_root_dir
    # Path to TB binary (relative or absolute)
    TB_BINARY_PATH = str(tb_root_dir / "bin" / "tbx")

    # Alternative paths to try if main path doesn't exist
    ALTERNATIVE_PATHS = [
        str(tb_root_dir / "tb-exc" / "target" / "debug" / "tbx"),
        str(tb_root_dir / "tb-exc" / "target" / "release" / "tbx"),
        #"tbx",  # System PATH
    ]

    # Add system-specific extension
    def add_extension(path: str) -> str:
        if system() == "Windows" and not path.endswith(".exe"):
            return f"{path}.exe"
        return path

    # Prepare paths with proper extensions
    paths_to_try = [add_extension(TB_BINARY_PATH)]
    for alt_path in ALTERNATIVE_PATHS:
        paths_to_try.append(add_extension(alt_path))

    for path in paths_to_try:
        # Check if file exists directly
        if os.path.exists(path):
            return path

        # Use shutil.which for system PATH lookup (cross-platform)
        if shutil.which(path):
            return path

    print(f"{Colors.RED}✗ TB binary not found!{Colors.RESET}")
    print(f"{Colors.YELLOW}Tried paths:{Colors.RESET}")
    for path in paths_to_try:
        print(f"  • {path}")
    print(f"\n{Colors.CYAN}Build the binary with:{Colors.RESET}")
    print(f"  tb run build")
    return ""

run_tb(code, mode='jit', timeout=10)

Run TB code and return (success, stdout, stderr).

Parameters:

Name	Type	Description	Default
code	str	TB source code	required
mode	str	Execution mode (jit, compiled, streaming)	'jit'
timeout	int	Timeout in seconds	10

Returns:

Type	Description
Tuple[bool, str, str]	(success, stdout, stderr)

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

def run_tb(code: str, mode: str = "jit", timeout: int = 10) -> Tuple[bool, str, str]:
    """
    Run TB code and return (success, stdout, stderr).

    Args:
        code: TB source code
        mode: Execution mode (jit, compiled, streaming)
        timeout: Timeout in seconds

    Returns:
        (success, stdout, stderr)
    """
    if mode == "compiled":
        with tempfile.NamedTemporaryFile(suffix='', delete=False) as f:
            output_path = f.name

        try:
            start = time.perf_counter()
            success, stderr = run_tb_compile(code, output_path)
            duration = time.perf_counter() - start
            print(f" -- Compile time ({duration:.3f}s)")
            if not success:
                raise AssertionError(f"Compilation failed:\n{stderr}")

            # Check binary exists
            if not os.path.exists(output_path):
                raise AssertionError("Compiled binary not found")

            # Check binary is executable
            if not os.access(output_path, os.X_OK):
                os.chmod(output_path, 0o755)

            start = time.perf_counter()
            # Run compiled binary
            result = subprocess.run([output_path], capture_output=True, text=True, timeout=timeout//2,
                                    encoding=sys.stdout.encoding or 'utf-8')
            duration = time.perf_counter() - start
            print(f" -- Exec time ({duration:.3f}s)")

            if result.returncode != 0:
                raise AssertionError(f"Compiled binary failed: {result.stderr}")

            return success, result.stdout, result.stderr
        finally:
            if os.path.exists(output_path):
                os.remove(output_path)

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, encoding=sys.stdout.encoding or 'utf-8') as f:
        f.write(code)
        temp_file = f.name

    try:
        result = subprocess.run(
            [TB_BINARY, "run", temp_file, "--mode", mode],
            capture_output=True,
            text=True,
            timeout=timeout,
            encoding=sys.stdout.encoding or 'utf-8',
            errors='replace'
        )

        success = result.returncode == 0
        return success, result.stdout, result.stderr

    except subprocess.TimeoutExpired:
        return False, "", f"Timeout after {timeout}s"

    finally:
        try:
            os.unlink(temp_file)
        except:
            pass

run_tb_compile(code, output_path, target=None)

Compile TB code to binary.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

def run_tb_compile(code: str, output_path: str, target: str = None) -> Tuple[bool, str]:
    """Compile TB code to binary."""
    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, encoding=sys.stdout.encoding or 'utf-8') as f:
        f.write(code)
        temp_file = f.name

    try:
        cmd = [TB_BINARY, "compile", temp_file, output_path]
        if target:
            cmd.extend(["--target", target])

        result = subprocess.run(cmd, capture_output=not VERBOSE, text=True, timeout=60, encoding=sys.stdout.encoding or 'utf-8')

        success = result.returncode == 0
        return success, result.stderr

    finally:
        try:
            os.unlink(temp_file)
        except:
            pass

test(name, category='General')

Decorator for test functions.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

def test(name: str, category: str = "General"):
    """Decorator for test functions."""

    def decorator(func):
        def wrapper():
            # Check filter
            if FILTER and FILTER.lower() not in name.lower():
                return

            # Print test header
            if suite.current_category != category:
                print(f"\n{Colors.BOLD}{Colors.CYAN}[{category}]{Colors.RESET}")
                suite.current_category = category

            print(f"  {Colors.GRAY}Testing:{Colors.RESET} {name}", end=" ", flush=True)

            start = time.perf_counter()
            try:
                func()
                duration = time.perf_counter() - start

                print(f" -- {Colors.GREEN}✓{Colors.RESET} ({duration:.3f}s)")

                suite.add_result(TestResult(
                    name=name,
                    passed=True,
                    duration_ms=duration
                ))

            except AssertionError as e:
                duration = time.perf_counter() - start

                print(f"{Colors.RED}✗{Colors.RESET} ({duration:.0f}s)")
                if VERBOSE:
                    print(f"    {Colors.RED}Error: {str(e)}{Colors.RESET}")

                suite.add_result(TestResult(
                    name=name,
                    passed=False,
                    duration_ms=duration,
                    error_message=str(e)
                ))

            except Exception as e:
                import traceback
                print(traceback.format_exc())
                duration = (time.perf_counter() - start) * 1000

                print(f"{Colors.RED}✗ (Exception){Colors.RESET}")
                if VERBOSE:
                    print(f"    {Colors.RED}{type(e).__name__}: {str(e)}{Colors.RESET}")

                suite.add_result(TestResult(
                    name=name,
                    passed=False,
                    duration_ms=duration,
                    error_message=f"{type(e).__name__}: {str(e)}"
                ))

        return wrapper

    return decorator

test_compiled_arithmetic()

Test basic arithmetic in compiled mode

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Compiled - Basic Arithmetic", "Compiled Modes")
def test_compiled_arithmetic():
    """Test basic arithmetic in compiled mode"""
    code = '''
let x = 10
let y = 20
let sum = x + y
let product = x * y
echo sum
echo product
'''
    success, stdout, stderr = run_tb(code, mode="compiled")
    assert success, f"Compilation failed: {stderr}"
    assert "30" in stdout
    assert "200" in stdout

test_compiled_functions()

Test function definitions in compiled mode

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Compiled - Functions", "Compiled Modes")
def test_compiled_functions():
    """Test function definitions in compiled mode"""
    code = '''
fn fibonacci(n: int) -> int {
    if n <= 1 {
        n
    } else {
        fibonacci(n - 1) + fibonacci(n - 2)
    }
}

let result = fibonacci(10)
echo result
'''
    assert_output(code, "55", mode="compiled")

test_compiled_functions_no_compiled_()

Test function definitions in compiled mode

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Compiled - Functions", "Compiled Modes NO Compiled")
def test_compiled_functions_no_compiled_():
    """Test function definitions in compiled mode"""
    code = '''
fn fibonacci(n: int) -> int {
    if n <= 1 {
        n
    } else {
        fibonacci(n - 1) + fibonacci(n - 2)
    }
}

let result = fibonacci(10)
echo result
'''
    assert_output(code, "55", mode="jit")

test_compiled_loops()

Test loops in compiled mode

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Compiled - Loops", "Compiled Modes")
def test_compiled_loops():
    """Test loops in compiled mode"""
    code = '''
let sum = 0
for i in [1, 2, 3, 4, 5] {
    sum = sum + i
}
echo sum
'''
    assert_output(code, "15", mode="compiled")

test_compiled_parallel()

Test parallel execution in compiled mode

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Compiled - Parallel Execution", "Compiled Modes")
def test_compiled_parallel():
    """Test parallel execution in compiled mode"""
    code = '''
@config {
    runtime_mode: "parallel"
}

let results = parallel {
    10 + 5,
    20 * 2,
    30 - 10
}
for result in results {
    echo result
}
'''
    success, stdout, stderr = run_tb(code, mode="compiled")
    assert success, f"Compilation failed: {stderr}"
    # Results may be in any order due to parallelism
    assert "15" in stdout, f"result {stdout}"
    assert "40" in stdout, f"result {stdout}"
    assert "20" in stdout, f"result {stdout}"

test_compiled_parallel_with_imports()

Test parallel execution with imported functions

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Compiled - Parallel with Imports", "Compiled Modes")
def test_compiled_parallel_with_imports():
    """Test parallel execution with imported functions"""
    lib_code = '''
fn compute_square(x: int) {
    x * x
}

fn compute_cube(x: int) {
    x * x * x
}
'''

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as lib_file:
        lib_file.write(lib_code)
        lib_path = os.path.basename(lib_file.name)

    try:
        main_code = f'''
@config {{
    runtime_mode: "parallel"
}}

@imports {{
    "{lib_path}"
}}

let results = parallel {{
    compute_square(5),
    compute_cube(3),
    compute_square(10)
}}

for result in results {{
    echo result
}}
'''
        success, stdout, stderr = run_tb(main_code, mode="compiled")
        assert success, f"Compilation failed: {stderr}"
        assert "25" in stdout
        assert "27" in stdout
        assert "100" in stdout
    finally:
        os.unlink(lib_path)

test_go_basic_execution()

Test that Go code can be executed.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Go Language Bridge - Basic Execution", "Dependencies - Go Bridge")
def test_go_basic_execution():
    """Test that Go code can be executed."""
    code = '''
let result = go("""
import "fmt"
fmt.Println("Hello from Go!")
""")
'''
    assert_success(code)

test_go_bridge_compiled_mode()

Test that the Go language bridge works in compiled mode.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Go Bridge - Compiled Mode", "Dependencies - Go Bridge")
def test_go_bridge_compiled_mode():
    """Test that the Go language bridge works in compiled mode."""
    code = '''
@config {
    mode: "compiled"
}

let result = go("""
import "fmt"
fmt.Println("Hello from Go in compiled mode!")
""")

echo "Go execution finished."
'''
    assert_contains(code, "Go execution finished.", mode="compiled")

test_go_import_json()

Test marshalling data to JSON using Go's 'encoding/json' package.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Go Import - JSON Marshalling", "Dependencies - Go Bridge")
def test_go_import_json():
    """Test marshalling data to JSON using Go's 'encoding/json' package."""
    code = '''
# This test assumes TB lists/dicts are converted to Go slices/maps
let user_data = [
    { "name": "Alice", "age": 30 },
    { "name": "Bob", "age": 25 }
]
let result = go("""
import (
    "fmt"
    "encoding/json"
)
// user_data is available from TB, assume it's a []map[string]interface{}
// Marshal returns the JSON encoding of user_data.
jsonData, err := json.Marshal(user_data)
if err != nil {
    fmt.Println("Error:", err)
} else {
    fmt.Println(string(jsonData))
}
""")
'''
    assert_success(code)

test_go_import_math()

Test using the Go 'math' standard library package.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Go Import - Math", "Dependencies - Go Bridge")
def test_go_import_math():
    """Test using the Go 'math' standard library package."""
    code = '''
let result = go("""
import (
    "fmt"
    "math"
)
// Sqrt returns the square root of 64
fmt.Println(math.Sqrt(64))
""")
'''
    assert_success(code)

test_go_import_strings()

Test using the Go 'strings' package with a TB variable.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Go Import - Strings", "Dependencies - Go Bridge")
def test_go_import_strings():
    """Test using the Go 'strings' package with a TB variable."""
    code = '''
let my_string = "Hello, Go!"
let result = go("""
import (
    "fmt"
    "strings"
)
// my_string is available from TB context
// ToUpper returns a copy of the string with all letters mapped to their upper case.
fmt.Println(strings.ToUpper(my_string))
""")
'''
    assert_success(code)

test_go_multilang_python_to_go()

Test data flow from a Python block to a Go block.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Go Multi-Language - Python to Go", "Dependencies - Go Bridge")
def test_go_multilang_python_to_go():
    """Test data flow from a Python block to a Go block."""
    code = '''
# Python creates a string
let data_from_py = python("""
print("Data generated by Python")
""")

# Go processes the string from the TB variable
let go_result = go("""
import "fmt"
// data_from_py is available from TB context
// Printf formats according to a format specifier and writes to standard output.
fmt.Printf("Go received: '%s'\\n", data_from_py)
""")
'''
    assert_success(code)

test_import_cache_invalidation()

Test that cache is invalidated when source changes

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Compiled - Cache Invalidation", "Import Compiled Modes")
def test_import_cache_invalidation():
    """Test that cache is invalidated when source changes"""
    print("→ Test: Cache Invalidation")

    lib_path = 'test_cache_lib.tbx'

    # Write initial version
    with open(lib_path, 'w') as f:
        f.write('''
@config {
    mode: "compiled"
}

fn get_value() {
    42
}
''')

    try:
        main_code = f'''
@imports {{
    "{lib_path}"
}}

echo get_value()
'''

        # First run
        success1, stdout1, _ = run_tb(main_code)
        assert success1
        assert "42" in stdout1

        # Modify library
        time.sleep(0.1)  # Ensure timestamp difference
        with open(lib_path, 'w') as f:
            f.write('''
@config {
    mode: "compiled"
}

fn get_value() {
    100
}
''')

        # Second run should use updated version
        success2, stdout2, _ = run_tb(main_code)
        assert success2
        assert "100" in stdout2, f"Cache not invalidated! Got: {stdout2}"

        print("  ✓ Cache invalidation works")

    finally:
        os.unlink(lib_path)

test_import_compiled_caching()

Test that compiled imports are cached

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Compiled - Import Caching", "Import Compiled Modes")
def test_import_compiled_caching():
    """Test that compiled imports are cached"""
    print("→ Test: Compiled Import Caching")

    # Create library with compiled mode
    lib_code = '''
@config {
    mode: "compiled"
}

fn factorial(n: int) {
    if n <= 1 {
        1
    } else {
        n * factorial(n - 1)
    }
}
'''

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as lib_file:
        lib_file.write(lib_code)
        lib_path = os.path.basename(lib_file.name)

    try:
        main_code = f'''

@config {{
    mode: "compiled"
}}

@imports {{
    "{lib_path}"
}}

let result = factorial(5)
echo $result
'''

        # First run - should compile
        start_time = time.perf_counter()
        success1, stdout1, stderr1 = run_tb(main_code)
        first_run_time = time.perf_counter() - start_time

        assert success1, f"First execution failed: {stderr1}"
        assert "120" in stdout1, f"Expected 120, got: {stdout1}"

        # Second run - should use cache
        start_time = time.perf_counter()
        success2, stdout2, stderr2 = run_tb(main_code)
        second_run_time = time.perf_counter() - start_time

        assert success2, f"Second execution failed: {stderr2}"
        assert "120" in stdout2, f"Expected 120, got: {stdout2}"

        # Second run should be significantly faster (using cache)
        assert second_run_time <= first_run_time, \
            f"Cache not used? First: {first_run_time:.2f}s, Second: {second_run_time:.2f}s"

        print(f"  ✓ Caching works (first: {first_run_time:.2f}s, second: {second_run_time:.2f}s)")

    finally:
        os.unlink(lib_path)

test_import_compiled_mode()

Test that imports work in compiled mode

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Import - Compiled Mode", "Import System")
def test_import_compiled_mode():
    """Test that imports work in compiled mode"""
    lib_code = '''
@config {
    mode: "compiled"
    target: "library"
}

fn double(x: int) {
    x * 2
}
'''

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as lib_file:
        lib_file.write(lib_code)
        lib_path = os.path.basename(lib_file.name)

    try:
        main_code = f'''
@config {{
    mode: "compiled"
}}

@imports {{
    "{lib_path}"
}}

let result = double(21)
echo result
'''
        assert_output(main_code, "42", mode="compiled")
    finally:
        os.unlink(lib_path)

test_import_dependency_chain()

Test imports that depend on other imports

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Compiled - Import Dependency Chain", "Import Compiled Modes")
def test_import_dependency_chain():
    """Test imports that depend on other imports"""
    print("→ Test: Import Dependency Chain")

    # Base library
    base_lib = '''
fn double(x: int) {
    x * 2
}
'''

    # Mid library (uses base)
    mid_lib = '''
@imports {
    "base.tbx"
}

fn quadruple(x: int) {
    double(double(x))
}
'''

    # Create base.tbx
    with open('base.tbx', 'w') as f:
        f.write(base_lib)

    # Create mid.tbx
    with open('mid.tbx', 'w') as f:
        f.write(mid_lib)

    try:
        main_code = '''
@imports {
    "mid.tbx"
}

echo quadruple(5)
'''

        success, stdout, stderr = run_tb(main_code)

        assert success, f"Execution failed: {stderr}"
        assert "20" in stdout, f"Expected 20, got: {stdout}"

        print("  ✓ Transitive imports work")

    finally:
        os.unlink('base.tbx')
        os.unlink('mid.tbx')

test_import_jit_vs_compiled()

Test that JIT imports don't get compiled

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Compiled - JIT vs Compiled Imports", "Import Compiled Modes")
def test_import_jit_vs_compiled():
    """Test that JIT imports don't get compiled"""
    print("→ Test: JIT vs Compiled Imports")

    # JIT library
    jit_lib = '''
@config {
    mode: "jit"
}

fn square(x: int) {
    x * x
}
'''

    # Compiled library
    compiled_lib = '''
@config {
    mode: "compiled"
}

fn cube(x: int) {
    x * x * x
}
'''

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as jit_file, \
         tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as compiled_file:

        jit_file.write(jit_lib)
        compiled_file.write(compiled_lib)
        jit_path = os.path.basename(jit_file.name)
        compiled_path = os.path.basename(compiled_file.name)

    try:
        main_code = f'''
@imports {{
    "{jit_path}"
    "{compiled_path}"
}}

echo square(4)
echo cube(3)
'''

        success, stdout, stderr = run_tb(main_code)

        assert success, f"Execution failed: {stderr}"
        assert "16" in stdout, f"Expected 16 from square(4), got: {stdout}"
        assert "27" in stdout, f"Expected 27 from cube(3), got: {stdout}"

        print("  ✓ Mixed JIT/Compiled imports work")

    finally:
        os.unlink(jit_path)
        os.unlink(compiled_path)

test_import_missing_file()

Test error handling for missing import files

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Import - Missing File Error", "Import Errors")
def test_import_missing_file():
    """Test error handling for missing import files"""
    code = '''
@imports {
    "nonexistent_file.tbx"
}

echo "This should not run"
'''
    success, stdout, stderr = run_tb(code)
    assert not success, "Should fail for missing import file"
    assert "not found" in stderr.lower() or "import" in stderr.lower()

test_import_multiple_files()

Test importing multiple .tbx files

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Import - Multiple Files", "Import System")
def test_import_multiple_files():
    """Test importing multiple .tbx files"""
    # Create math library
    math_lib = '''
fn add(x, y) {
    x + y
}
'''

    # Create string library
    string_lib = '''
fn greet(name: string) {
    echo "Hello,"name "!"
}
'''

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as math_file, \
        tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as string_file:

        math_file.write(math_lib)
        string_file.write(string_lib)
        math_path = os.path.basename(math_file.name)
        string_path = os.path.basename(string_file.name)

    try:
        main_code = f'''
@imports {{
    "{math_path}"
    "{string_path}"
}}

let sum = add(3, 7)
echo sum
greet("World")
'''
        success, stdout, stderr = run_tb(main_code)
        assert success, f"Execution failed: {stderr}"
        assert "10" in stdout, f"Expected '10' in output, got: {stdout}"
        assert "Hello, World !" in stdout, f"Expected Hello, World! in output, got: {stdout}"
    finally:
        os.unlink(math_path)
        os.unlink(string_path)

test_import_no_circular()

Test that circular imports are handled (imports are not recursive)

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Import - Circular Import Protection", "Import Errors")
def test_import_no_circular():
    """Test that circular imports are handled (imports are not recursive)"""
    # Create file that tries to import itself
    lib_code = '''
fn test_func() {
    echo "Hello"
}
'''

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as lib_file:
        lib_file.write(lib_code)
        lib_path = os.path.basename(lib_file.name)

    try:
        # This should work because imports are not recursive
        main_code = f'''
@imports {{
    "{lib_path}"
}}

test_func()
'''
        assert_success(main_code)
    finally:
        os.unlink(lib_path)

test_import_relative_paths()

Test importing with relative directory paths

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Import - Relative Paths", "Import System")
def test_import_relative_paths():
    """Test importing with relative directory paths"""
    # Create subdirectory
    lib_dir = tempfile.mkdtemp(dir='..')
    lib_dir_name = os.path.basename(lib_dir)

    try:
        # Create library in subdirectory
        lib_code = '''
fn multiply(x: int, y: int) {
    x * y
}
'''
        lib_path = os.path.join(lib_dir, 'math.tbx')
        with open(lib_path, 'w') as f:
            f.write(lib_code)

        main_code = f'''
@imports {{
    "{lib_dir_name}/math.tbx"
}}

let result = multiply(6, 7)
echo result
'''
        assert_output(main_code, "42")
    finally:
        shutil.rmtree(lib_dir, ignore_errors=True)

test_import_single_file()

Test importing a single .tbx file

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Import - Single File", "Import System")
def test_import_single_file():
    """Test importing a single .tbx file"""
    # Create library file
    lib_code = '''
fn double(x: int) {
    x * 2
}

fn triple(x: int) {
    x * 3
}
'''

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as lib_file:
        lib_file.write(lib_code)
        lib_path = os.path.basename(lib_file.name)

    try:
        # Create main file that imports library
        main_code = f'''
@imports {{
    "{lib_path}"
}}

let result = double(5)
echo result
'''
        assert_output(main_code, "10")
    finally:
        os.unlink(lib_path)

test_import_with_variables()

Test importing file with global variables

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Import - With Variables", "Import System")
def test_import_with_variables():
    """Test importing file with global variables"""
    lib_code = '''
let PI = 3.14159
let E = 2.71828

fn circle_area(radius: float) {
    PI * radius * radius
}
'''

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as lib_file:
        lib_file.write(lib_code)
        lib_path = os.path.basename(lib_file.name)

    try:
        main_code = f'''
@imports {{
    "{lib_path}"
}}

echo PI
let area = circle_area(2.0)
echo area
'''
        success, stdout, stderr = run_tb(main_code)
        assert success, f"Execution failed: {stderr}"
        assert "3.14159" in stdout or "3.14" in stdout
    finally:
        os.unlink(lib_path)

test_mixed_full_stack()

Comprehensive test with all features combined

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Mixed - Full Stack Test", "Mixed Features")
def test_mixed_full_stack():
    """Comprehensive test with all features combined"""
    helpers_lib = '''
fn compute(x: int) {
    x * 2
}

fn parallel_sum(numbers: list) {
    let sum = 0
    for n in numbers {
        sum = sum + n
    }
    sum
}
'''

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as lib_file:
        lib_file.write(helpers_lib)
        lib_path = os.path.basename(lib_file.name)

    try:
        main_code = f'''
@config {{
    mode: "jit"
}}

@shared {{
    total: 0
}}

@imports {{
    "{lib_path}"
}}

let value = compute(21)
total = parallel_sum([1, 2, 3, 4, 5])
echo value
echo total
'''
        success, stdout, stderr = run_tb(main_code, mode="jit")
        assert success, f"Execution failed: {stderr}"
        assert "42" in stdout
        assert "15" in stdout
    finally:
        os.unlink(lib_path)

test_mixed_import_async_language()

Test imports with async and language bridges

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Mixed - Import  + Language Bridge", "Mixed Features")
def test_mixed_import_async_language():
    """Test imports with async and language bridges"""
    lib_code = '''
fn get_data() {

    python("import sys; print(sys.version.split()[0])")

}
'''

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as lib_file:
        lib_file.write(lib_code)
        lib_path = os.path.basename(lib_file.name)

    try:
        main_code = f'''

@imports {{
    "{lib_path}"
}}

let version = get_data()
echo "Python version detected"
'''
        assert_contains(main_code, "Python version detected", mode="jit")
    finally:
        os.unlink(lib_path)

test_mixed_multiple_imports_parallel()

Test multiple imports with parallel execution

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Mixed - Multiple Imports + Parallel", "Mixed Features")
def test_mixed_multiple_imports_parallel():
    """Test multiple imports with parallel execution"""
    math_lib = '''
fn factorial(n: int) -> int {
    if n <= 1 {
        1
    } else {
        n * factorial(n - 1)
    }
}
'''

    utils_lib = '''
fn power(base: int, exp: int) -> int {
    if exp == 0 {
        1
    } else {
        base * power(base, exp - 1)
    }
}
'''

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as math_file, \
        tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as utils_file:

        math_file.write(math_lib)
        utils_file.write(utils_lib)
        math_path = os.path.basename(math_file.name)
        utils_path = os.path.basename(utils_file.name)

    try:
        main_code = f'''
@config {{
    runtime_mode: "parallel"
}}

@imports {{
    "{math_path}"
    "{utils_path}"
}}

let results = parallel {{
    factorial(5),
    power(2, 10),
    factorial(6)
}}

for result in results {{
    echo result
}}
'''
        success, stdout, stderr = run_tb(main_code, mode="compiled")
        assert success, f"Compilation failed: {stderr}"
        assert "120" in stdout  # 5!
        assert "1024" in stdout  # 2^10
        assert "720" in stdout  # 6!
    finally:
        os.unlink(math_path)
        os.unlink(utils_path)

test_mixed_shared_imports()

Test shared variables with imports

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Mixed - Shared Variables + Imports", "Mixed Features")
def test_mixed_shared_imports():
    """Test shared variables with imports"""
    lib_code = '''
fn increment_counter() {
    counter + 1
}
'''

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as lib_file:
        lib_file.write(lib_code)
        lib_path = os.path.basename(lib_file.name)

    try:
        main_code = f'''
@shared {{
    counter: 0
}}

@imports {{
    "{lib_path}"
}}

counter = increment_counter()
counter = increment_counter()
echo counter
'''
        assert_output(main_code, "2")
    finally:
        os.unlink(lib_path)

test_mixed_shared_imports_mixed_lang()

Test shared variables with imports

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Mixed - Shared Variables + Imports + Python Bridge", "Mixed Features")
def test_mixed_shared_imports_mixed_lang():
    """Test shared variables with imports"""
    lib_code = '''
fn increment_counter() {python("print(counter + 1, end='')")}

'''

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as lib_file:
        lib_file.write(lib_code)
        lib_path = os.path.basename(lib_file.name)

    try:
        main_code = f'''
@shared {{
    counter: 0
}}

@imports {{
    "{lib_path}"
}}

counter = increment_counter()
counter = increment_counter()
echo counter
'''
        assert_output(main_code, "2")
    finally:
        os.unlink(lib_path)

test_type_annotations()

Test that type annotations are correctly handled.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Type Annotations - Basic Types", "Type Annotations")
def test_type_annotations():
    """Test that type annotations are correctly handled."""
    code = '''@config { mode: "jit", type_system: "static" }

let age: int = python("print(42)")
let price: float = python("print(19.99)")
let name: string = python("""x = "Alice"
x""")
let active: bool = python("print(True)")
let scores: list = python("print([85, 92, 78])")

echo "Age: $age"           // Age: 42
echo "Price: $price"       // Price: 19.99
echo "Name: $name"         // Name: Alice
echo "Active: $active"     // Active: true'''
    assert_output(code, "Age: 42\nPrice: 19.99\nName: Alice\nActive: true")

test_type_annotations_()

Test that type annotations are correctly handled.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Type Annotations - Basic Types", "Type Annotations")
def test_type_annotations_():
    """Test that type annotations are correctly handled."""
    code = '''@config { mode: "jit", type_system: "static" }

let age: int = python("print(42)")
let price: float = python("19.99")
let name: string = python("Alice")
let active: bool = python("True")
let scores: list<int> = python("[85, 92, 78]")

echo "Age: $age"           // Age: 42
echo "Price: $price"       // Price: 19.99
echo "Name: $name"         // Name: Alice
echo "Active: $active"     // Active: true'''
    assert_output(code, "Age: 42\nPrice: 19.99\nName: Alice\nActive: true")

test_type_annotations_auto()

Test that type annotations are correctly handled.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Type Annotations - Auto Type Inference", "Type Annotations")
def test_type_annotations_auto():
    """Test that type annotations are correctly handled."""
    code = '''@config { mode: "jit", type_system: "static" }

let age: int = python("print(42)")
let price: float = python("print(19.99)")
let name: string = python("print('Alice')")
let active: bool = python("print(True)")
let scores: list = python("print([85, 92, 78])")

echo type_of(age)           // Age: 42
echo type_of(price)       // Price: 19.99
echo type_of(name)         // Name: Alice
echo type_of(active)         // Active: true
echo type_of(scores)     // Scores: [85, 92, 78]'''
    assert_output(code, "int\nfloat\nstring\nbool\nlist")

test_type_annotations_auto_compiled()

Test that type annotations are correctly handled.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Type Annotations - Auto Type Inference  - Compiled", "Type Annotations - Compiled")
def test_type_annotations_auto_compiled():
    """Test that type annotations are correctly handled."""
    code = '''@config { mode: "compiled", type_system: "static" }

let age: int = python("print(42)")
let price: float = python("print(19.99)")
let name: string = python("print('Alice')")
let active: bool = python("print(True)")
let scores: list<int> = python("print([85, 92, 78])")

echo type_of($age)           // Age: 42
echo type_of($price)       // Price: 19.99
echo type_of($name)         // Name: Alice
echo type_of(active)         // Active: true
echo type_of(scores)     // Scores: [85, 92, 78]'''
    assert_output(code, "int\nfloat\nstring\nbool\nlist<int>", mode="compiled")

test_type_annotations_compiled()

Test that type annotations are correctly handled.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Type Annotations - Basic Type - Compiled", "Type Annotations - Compiled")
def test_type_annotations_compiled():
    """Test that type annotations are correctly handled."""
    code = '''@config { mode: "compiled", type_system: "static" }

let age: int = python("print(42)")
let price: float = python("print(19.99)")
let name: string = python("print('Alice')")
let active: bool = python("print(True)")
let scores: list = python("print([85, 92, 78])")

echo "Age: $age"           // Age: 42
echo "Price: $price"       // Price: 19.99
echo "Name: $name"         // Name: Alice
echo "Active: $active"     // Active: true'''
    assert_output(code, "Age: 42\nPrice: 19.99\nName: Alice\nActive: true", mode="compiled")

test_tb_lang2

TB Language Comprehensive Test Suite Tests all features of the TB language implementation.

Usage

python test_tb_lang.py python test_tb_lang.py --verbose python test_tb_lang.py --filter "test_arithmetic" python test_tb_lang.py --mode jit python test_tb_lang.py --mode compiled python test_tb_lang.py --skip-slow

TestSuite

Source code in toolboxv2/utils/tbx/test/test_tb_lang2.py

class TestSuite:
    def __init__(self):
        self.results: List[TestResult] = []
        self.current_category = ""
        self.failed_filter = None
        self.failed_tests_cache = self.load_failed_tests()

    def load_failed_tests(self) -> set:
        """Load previously failed test names from cache."""
        cache_file = Path(__file__).parent / ".failed_tests.json"
        if cache_file.exists():
            try:
                with open(cache_file, 'r') as f:
                    data = json.load(f)
                    return set(data.get('failed_tests', []))
            except:
                pass
        return set()

    def save_failed_tests(self):
        """Save failed test names to cache."""
        cache_file = Path(__file__).parent / ".failed_tests.json"
        failed_names = [r.name for r in self.results if not r.passed]
        with open(cache_file, 'w') as f:
            json.dump({'failed_tests': failed_names}, f, indent=2)

    def should_run_test(self, test_name: str) -> bool:
        """Check if test should run based on FAILED_ONLY flag."""
        if not FAILED_ONLY:
            return True
        return test_name in self.failed_tests_cache

    def add_result(self, result: TestResult):
        self.results.append(result)

    def print_summary(self):
        total = len(self.results)
        passed = sum(1 for r in self.results if r.passed)
        failed = total - passed
        total_time = sum(r.duration_ms for r in self.results)

        print("\n" + "=" * 80)
        print(f"{Colors.BOLD}TEST SUMMARY{Colors.RESET}")
        print("=" * 80)

        if failed == 0:
            print(f"{Colors.GREEN}OK - All {total} tests passed!{Colors.RESET}")
        else:
            print(f"{Colors.RED}FAILED - {failed} of {total} tests failed{Colors.RESET}")
            print(f"{Colors.GREEN}OK - {passed} passed{Colors.RESET}")

        print(f"\n{Colors.CYAN}Total time: {total_time:.2f}ms{Colors.RESET}")

        # Performance statistics
        jit_results = [r for r in self.results if r.mode == "jit" and r.passed]
        compiled_results = [r for r in self.results if r.mode == "compiled" and r.passed]

        if jit_results:
            avg_jit = sum(r.duration_ms for r in jit_results) / len(jit_results)
            print(f"{Colors.BLUE}JIT avg time: {avg_jit:.2f}ms{Colors.RESET}")

        if compiled_results:
            avg_compiled = sum(r.duration_ms for r in compiled_results) / len(compiled_results)
            avg_compile = sum(r.compile_time_ms for r in compiled_results if r.compile_time_ms) / len(compiled_results)
            avg_exec = sum(r.exec_time_ms for r in compiled_results if r.exec_time_ms) / len(compiled_results)
            print(
                f"{Colors.BLUE}Compiled avg time: {avg_compiled:.2f}ms (compile: {avg_compile:.2f}ms, exec: {avg_exec:.2f}ms){Colors.RESET}")

        if failed > 0:
            print(f"\n{Colors.RED}Failed tests:{Colors.RESET}")
            for result in self.results:
                if not result.passed:
                    print(f"  - {result.name} ({result.mode})")
                    if result.error_message:
                        # Encode error message safely to avoid Unicode issues
                        try:
                            print(f"    {Colors.GRAY}{result.error_message}{Colors.RESET}")
                        except UnicodeEncodeError:
                            # Fallback: print without special characters
                            safe_msg = result.error_message.encode('ascii', 'replace').decode('ascii')
                            print(f"    {Colors.GRAY}{safe_msg}{Colors.RESET}")

        return failed == 0

load_failed_tests()

Load previously failed test names from cache.

Source code in toolboxv2/utils/tbx/test/test_tb_lang2.py

def load_failed_tests(self) -> set:
    """Load previously failed test names from cache."""
    cache_file = Path(__file__).parent / ".failed_tests.json"
    if cache_file.exists():
        try:
            with open(cache_file, 'r') as f:
                data = json.load(f)
                return set(data.get('failed_tests', []))
        except:
            pass
    return set()

save_failed_tests()

Save failed test names to cache.

Source code in toolboxv2/utils/tbx/test/test_tb_lang2.py

def save_failed_tests(self):
    """Save failed test names to cache."""
    cache_file = Path(__file__).parent / ".failed_tests.json"
    failed_names = [r.name for r in self.results if not r.passed]
    with open(cache_file, 'w') as f:
        json.dump({'failed_tests': failed_names}, f, indent=2)

should_run_test(test_name)

Check if test should run based on FAILED_ONLY flag.

Source code in toolboxv2/utils/tbx/test/test_tb_lang2.py

def should_run_test(self, test_name: str) -> bool:
    """Check if test should run based on FAILED_ONLY flag."""
    if not FAILED_ONLY:
        return True
    return test_name in self.failed_tests_cache

assert_contains(code, substring, mode='jit')

Assert that output contains substring.

Source code in toolboxv2/utils/tbx/test/test_tb_lang2.py

def assert_contains(code: str, substring: str, mode: str = "jit"):
    """Assert that output contains substring."""
    success, stdout, stderr, compile_time, exec_time = run_tb(code, mode)

    if not success:
        raise AssertionError(f"Execution failed:\n{stderr}")

    if substring not in stdout:
        raise AssertionError(f"Output does not contain '{substring}':\n{stdout}")

assert_error(code, mode='jit')

Assert that code fails.

Source code in toolboxv2/utils/tbx/test/test_tb_lang2.py

def assert_error(code: str, mode: str = "jit"):
    """Assert that code fails."""
    success, stdout, stderr, compile_time, exec_time = run_tb(code, mode)

    if success:
        raise AssertionError(f"Expected failure but succeeded:\n{stdout}")

assert_output(code, expected, mode='jit')

Assert that TB code produces expected output.

Source code in toolboxv2/utils/tbx/test/test_tb_lang2.py

def assert_output(code: str, expected: str, mode: str = "jit"):
    """Assert that TB code produces expected output."""
    success, stdout, stderr, compile_time, exec_time = run_tb(code, mode)

    if not success:
        raise AssertionError(f"Execution failed:\n{stderr}")

    actual = stdout.strip()
    expected = expected.strip()

    if actual != expected:
        raise AssertionError(
            f"Output mismatch:\nExpected: {repr(expected)}\nGot: {repr(actual)}"
        )

assert_output_with_tcp_server(code, expected, mode='jit', host='localhost', port=8085)

Run code while a temporary TCP server is alive. The server accepts a single connection, reads once, then closes.

Source code in toolboxv2/utils/tbx/test/test_tb_lang2.py

def assert_output_with_tcp_server(code: str, expected: str, mode: str = "jit",
                                  host: str = "localhost", port: int = 8085):
    """
    Run code while a temporary TCP server is alive.
    The server accepts a single connection, reads once, then closes.
    """
    received = []

    def _server():
        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
            s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
            s.bind((host, port))
            s.listen(1)
            conn, addr = s.accept()
            with conn:
                data = conn.recv(4096)
                if data:
                    received.append(data)

    t = threading.Thread(target=_server, daemon=True)
    t.start()

    # run TB code
    success, stdout, stderr, compile_time, exec_time = run_tb(code, mode)

    if not success:
        raise AssertionError(f"Execution failed:\n{stderr}")

    actual = stdout.strip()
    expected = expected.strip()
    if actual != expected:
        raise AssertionError(
            f"Output mismatch:\nExpected: {repr(expected)}\nGot: {repr(actual)}"
        )

    # optionally validate something was actually received
    if not received:
        raise AssertionError("TCP server received no data")

assert_success(code, mode='jit')

Assert that TB code runs without error.

Source code in toolboxv2/utils/tbx/test/test_tb_lang2.py

def assert_success(code: str, mode: str = "jit"):
    """Assert that TB code runs without error."""
    success, stdout, stderr, compile_time, exec_time = run_tb(code, mode)

    if VERBOSE:
        print(f"\n    stdout: {stdout}")
        if stderr:
            print(f"    stderr: {stderr}")

    if not success:
        raise AssertionError(f"Execution failed:\n{stderr}")

escape_path_for_tb(path)

Escape path for TB string literals.

Source code in toolboxv2/utils/tbx/test/test_tb_lang2.py

def escape_path_for_tb(path):
    """Escape path for TB string literals."""
    return path.replace('\\', '\\\\')

find_tb_binary()

Find TB binary in multiple locations.

Source code in toolboxv2/utils/tbx/test/test_tb_lang2.py

def find_tb_binary() -> str:
    """Find TB binary in multiple locations."""
    try:
        from toolboxv2 import tb_root_dir
        paths = [
            tb_root_dir / "tb-exc" /"src" / "target" / "release" / "tb",  # Prefer release for faster compilation
            tb_root_dir / "tb-exc" /"src" / "target" / "debug" / "tb",
            tb_root_dir / "bin" / "tb",
        ]
    except:
        paths = [
            Path("target/release/tb"),
            Path("target/debug/tb"),
            Path("tb"),
        ]

    paths = [os.environ.get("TB_EXE"), os.environ.get("TB_BINARY")]+paths
    # Add .exe for Windows
    if os.name == 'nt':
        paths = [Path(str(p) + ".exe") for p in paths if p is not None]

    for path in paths:
        if path is None:
            continue
        if shutil.which(str(path)) or os.path.exists(path):
            return str(path)

    print(f"{Colors.YELLOW}Tried paths:{Colors.RESET}")
    for path in paths:
        print(f"  • {path}")
    print(f"\n{Colors.CYAN}Build with: tb run build{Colors.RESET}")

load_failed_tests()

Load failed test names from file.

Source code in toolboxv2/utils/tbx/test/test_tb_lang2.py

def load_failed_tests():
    """Load failed test names from file."""
    try:
        if os.path.exists(FAILED_TESTS_FILE):
            with open(FAILED_TESTS_FILE, 'r', encoding='utf-8') as f:
                return set(line.strip() for line in f if line.strip())
    except Exception as e:
        print(f"{Colors.YELLOW}Warning: Could not load failed tests: {e}{Colors.RESET}")
    return set()

save_failed_tests(failed_names)

Save failed test names to file.

Source code in toolboxv2/utils/tbx/test/test_tb_lang2.py

def save_failed_tests(failed_names):
    """Save failed test names to file."""
    try:
        with open(FAILED_TESTS_FILE, 'w', encoding='utf-8') as f:
            for name in failed_names:
                f.write(f"{name}\n")
    except Exception as e:
        print(f"{Colors.YELLOW}Warning: Could not save failed tests: {e}{Colors.RESET}")

tbx_setup

TB Language Setup Utility - File association (.tbx files) - Icon registration - Desktop integration

TBxSetup

Setup utility for TB Language file associations and icons

Source code in toolboxv2/utils/tbx/setup.py

class TBxSetup:
    """Setup utility for TB Language file associations and icons"""

    def __init__(self):
        self.system = platform.system()
        self.tb_root = self.get_tb_root()
        self.icon_path = Path(os.getenv("FAVI", ".ico"))
        self.executable = self.get_executable()

    def get_tb_root(self) -> Path:
        """Get toolbox root directory"""
        try:
            from toolboxv2 import tb_root_dir
            return Path(tb_root_dir)
        except ImportError:
            return Path(__file__).parent.parent

    def get_executable(self) -> Path:
        """Get TB executable path"""
        if self.system == "Windows":
            exe = self.tb_root / "bin" / "tb.exe"
        else:
            exe = self.tb_root / "bin" / "tb"

        if not exe.exists():
            # Try target/release
            if self.system == "Windows":
                exe = self.tb_root / "tb-exc" / "target" / "release" / "tb.exe"
            else:
                exe = self.tb_root / "tb-exc" / "target" / "release" / "tb"

        return exe

    def setup_all(self):
        """Run complete setup"""
        print("╔════════════════════════════════════════════════════════════════╗")
        print("║         TB Language - System Integration Setup                 ║")
        print("╚════════════════════════════════════════════════════════════════╝")
        print()

        # Check prerequisites
        if not self.executable.exists():
            print("❌ TB executable not found!")
            print(f"   Expected at: {self.executable}")
            print("   Run 'tb x build' first!")
            return False

        print(f"✓ TB executable found: {self.executable}")
        print()

        # Setup icon
        if not self.setup_icon():
            print("⚠️  Icon setup failed (continuing anyway)")

        # Setup file association
        if self.system == "Windows":
            success = self.setup_windows()
        elif self.system == "Linux":
            success = self.setup_linux()
        elif self.system == "Darwin":
            success = self.setup_macos()
        else:
            print(f"❌ Unsupported system: {self.system}")
            return False

        if success:
            print()
            print("╔════════════════════════════════════════════════════════════════╗")
            print("║                    ✓ Setup Complete!                           ║")
            print("╠════════════════════════════════════════════════════════════════╣")
            print("║  .tbx files are now associated with TB Language                ║")
            print("║  Double-click any .tbx file to run it!                         ║")
            print("╚════════════════════════════════════════════════════════════════╝")

        return success

    def setup_icon(self) -> bool:
        """Setup icon file"""
        print("📦 Setting up icon...")

        icon_dir = self.tb_root / "resources"
        icon_dir.mkdir(exist_ok=True)

        # Check if icon exists
        if self.icon_path.exists():
            print(f"   ✓ Icon already exists: {self.icon_path}")
            return True

        # Create placeholder icon info
        print(f"   ⚠️  Icon not found at: {self.icon_path}")
        print(f"   📝 Creating placeholder...")

        # Try to create a simple icon reference
        # User needs to provide actual tb_icon.ico file
        placeholder = icon_dir / "README_ICON.txt"
        placeholder.write_text("""
TB Language Icon
================

Place your icon files here:
- tb_icon.ico   (Windows)
- tb_icon.png   (Linux)
- tb_icon.icns  (macOS)

Recommended size: 256x256 px

You can use the ToolBox V2 logo/icon.
        """)

        print(f"   ℹ️  Place icon file at: {self.icon_path}")
        return False

    def setup_windows(self) -> bool:
        """Setup file association on Windows"""
        print("🪟 Setting up Windows file association...")

        try:
            import winreg

            # Create .tbx extension key
            print("   Creating registry entries...")

            # HKEY_CURRENT_USER\Software\Classes\.tbx
            with winreg.CreateKey(winreg.HKEY_CURRENT_USER, r"Software\Classes\.tbx") as key:
                winreg.SetValue(key, "", winreg.REG_SZ, "TBLanguageFile")
                print("   ✓ Registered .tbx extension")

            # HKEY_CURRENT_USER\Software\Classes\TBLanguageFile
            with winreg.CreateKey(winreg.HKEY_CURRENT_USER, r"Software\Classes\TBLanguageFile") as key:
                winreg.SetValue(key, "", winreg.REG_SZ, "TB Language Program")

                # Set icon
                if self.icon_path.exists():
                    icon_key = winreg.CreateKey(key, "DefaultIcon")
                    winreg.SetValue(icon_key, "", winreg.REG_SZ, str(self.icon_path))
                    print(f"   ✓ Set icon: {self.icon_path}")

                # Set open command
                command_key = winreg.CreateKey(key, r"shell\open\command")
                cmd = f'"{self.executable}" run "%1"'
                winreg.SetValue(command_key, "", winreg.REG_SZ, cmd)
                print(f"   ✓ Set open command: {cmd}")

                # Add "Run in Terminal" context menu
                terminal_key = winreg.CreateKey(key, r"shell\run_terminal\command")
                terminal_cmd = f'cmd /k "{self.executable}" run "%1" && pause'
                winreg.SetValue(terminal_key, "", winreg.REG_SZ, terminal_cmd)
                winreg.SetValue(winreg.CreateKey(key, r"shell\run_terminal"), "", winreg.REG_SZ, "Run in Terminal")
                print(f"   ✓ Added 'Run in Terminal' context menu")

                # Add "Edit" context menu
                edit_key = winreg.CreateKey(key, r"shell\edit\command")
                winreg.SetValue(edit_key, "", winreg.REG_SZ, 'notepad "%1"')
                winreg.SetValue(winreg.CreateKey(key, r"shell\edit"), "", winreg.REG_SZ, "Edit")
                print(f"   ✓ Added 'Edit' context menu")

            # Refresh shell
            print("   Refreshing Explorer...")
            try:
                import ctypes
                ctypes.windll.shell32.SHChangeNotify(0x08000000, 0x0000, None, None)
            except:
                print("   ⚠️  Could not refresh Explorer (restart may be needed)")

            print("   ✓ Windows setup complete!")
            return True

        except ImportError:
            print("   ❌ winreg module not available")
            return False
        except Exception as e:
            print(f"   ❌ Error: {e}")
            return False

    def setup_linux(self) -> bool:
        """Setup file association on Linux"""
        print("🐧 Setting up Linux file association...")

        try:
            # Create .desktop file
            desktop_dir = Path.home() / ".local" / "share" / "applications"
            desktop_dir.mkdir(parents=True, exist_ok=True)

            desktop_file = desktop_dir / "tb-language.desktop"

            icon_path = self.icon_path.with_suffix('.png')
            if not icon_path.exists():
                icon_path = "text-x-script"  # Fallback icon

            desktop_content = f"""[Desktop Entry]
Version=1.0
Type=Application
Name=TB Language
Comment=Execute TB Language programs
Exec={self.executable} run %f
Icon={icon_path}
Terminal=false
MimeType=text/x-tb;application/x-tb;
Categories=Development;
"""

            desktop_file.write_text(desktop_content)
            desktop_file.chmod(0o755)
            print(f"   ✓ Created desktop entry: {desktop_file}")

            # Create MIME type
            mime_dir = Path.home() / ".local" / "share" / "mime" / "packages"
            mime_dir.mkdir(parents=True, exist_ok=True)

            mime_file = mime_dir / "tb-language.xml"
            mime_content = """<?xml version="1.0" encoding="UTF-8"?>
<mime-info xmlns="http://www.freedesktop.org/standards/shared-mime-info">
    <mime-type type="text/x-tb">
        <comment>TB Language Program</comment>
        <glob pattern="*.tbx"/>
        <sub-class-of type="text/plain"/>
    </mime-type>
</mime-info>
"""

            mime_file.write_text(mime_content)
            print(f"   ✓ Created MIME type: {mime_file}")

            # Update MIME database
            print("   Updating MIME database...")
            try:
                subprocess.run(["update-mime-database",
                                str(Path.home() / ".local" / "share" / "mime")],
                               check=True, capture_output=True)
                print("   ✓ MIME database updated")
            except:
                print("   ⚠️  Could not update MIME database automatically")
                print("   Run: update-mime-database ~/.local/share/mime")

            # Update desktop database
            print("   Updating desktop database...")
            try:
                subprocess.run(["update-desktop-database", str(desktop_dir)],
                               check=True, capture_output=True)
                print("   ✓ Desktop database updated")
            except:
                print("   ⚠️  Could not update desktop database automatically")

            # Set default application
            try:
                subprocess.run([
                    "xdg-mime", "default", "tb-language.desktop", "text/x-tb"
                ], check=True, capture_output=True)
                print("   ✓ Set as default application for .tbx files")
            except:
                print("   ⚠️  Could not set as default application")

            print("   ✓ Linux setup complete!")
            return True

        except Exception as e:
            print(f"   ❌ Error: {e}")
            return False

    def setup_macos(self) -> bool:
        """Setup file association on macOS"""
        print("🍎 Setting up macOS file association...")

        try:
            # Create Info.plist for file association
            app_dir = self.tb_root / "TB Language.app"
            contents_dir = app_dir / "Contents"
            macos_dir = contents_dir / "MacOS"
            resources_dir = contents_dir / "Resources"

            # Create directories
            macos_dir.mkdir(parents=True, exist_ok=True)
            resources_dir.mkdir(parents=True, exist_ok=True)

            # Copy executable
            app_executable = macos_dir / "tb"
            if not app_executable.exists():
                shutil.copy(self.executable, app_executable)
                app_executable.chmod(0o755)

            # Create launcher script
            launcher = macos_dir / "TB Language"
            launcher.write_text(f"""#!/bin/bash
if [ "$#" -gt 0 ]; then
    "{app_executable}" run "$@"
else
    "{app_executable}" repl
fi
""")
            launcher.chmod(0o755)

            # Create Info.plist
            plist_file = contents_dir / "Info.plist"
            plist_content = f"""<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
    <key>CFBundleExecutable</key>
    <string>TB Language</string>
    <key>CFBundleIconFile</key>
    <string>tb_icon</string>
    <key>CFBundleIdentifier</key>
    <string>dev.tblang.tb</string>
    <key>CFBundleName</key>
    <string>TB Language</string>
    <key>CFBundlePackageType</key>
    <string>APPL</string>
    <key>CFBundleShortVersionString</key>
    <string>1.0.0</string>
    <key>CFBundleVersion</key>
    <string>1.0.0</string>
    <key>CFBundleDocumentTypes</key>
    <array>
        <dict>
            <key>CFBundleTypeExtensions</key>
            <array>
                <string>tbx</string>
            </array>
            <key>CFBundleTypeIconFile</key>
            <string>tb_icon</string>
            <key>CFBundleTypeName</key>
            <string>TB Language Program</string>
            <key>CFBundleTypeRole</key>
            <string>Editor</string>
            <key>LSHandlerRank</key>
            <string>Owner</string>
        </dict>
    </array>
</dict>
</plist>
"""
            plist_file.write_text(plist_content)
            print(f"   ✓ Created app bundle: {app_dir}")

            # Copy icon if exists
            icon_src = self.icon_path.with_suffix('.icns')
            if icon_src.exists():
                shutil.copy(icon_src, resources_dir / "tb_icon.icns")
                print(f"   ✓ Copied icon")

            # Register with Launch Services
            print("   Registering with Launch Services...")
            try:
                subprocess.run([
                    "/System/Library/Frameworks/CoreServices.framework/Frameworks/LaunchServices.framework/Support/lsregister",
                    "-f", str(app_dir)
                ], check=True, capture_output=True)
                print("   ✓ Registered with Launch Services")
            except:
                print("   ⚠️  Could not register automatically")
                print(f"   Run: open '{app_dir}'")

            print("   ✓ macOS setup complete!")
            return True

        except Exception as e:
            print(f"   ❌ Error: {e}")
            return False

    def uninstall(self):
        """Remove file associations"""
        print("🗑️  Uninstalling file associations...")

        if self.system == "Windows":
            try:
                import winreg
                winreg.DeleteKey(winreg.HKEY_CURRENT_USER, r"Software\Classes\.tbx")
                winreg.DeleteKey(winreg.HKEY_CURRENT_USER, r"Software\Classes\TBLanguageFile")
                print("   ✓ Windows registry cleaned")
            except:
                print("   ⚠️  Could not clean registry")

        elif self.system == "Linux":
            desktop_file = Path.home() / ".local" / "share" / "applications" / "tb-language.desktop"
            mime_file = Path.home() / ".local" / "share" / "mime" / "packages" / "tb-language.xml"

            if desktop_file.exists():
                desktop_file.unlink()
                print("   ✓ Removed desktop entry")

            if mime_file.exists():
                mime_file.unlink()
                print("   ✓ Removed MIME type")

        elif self.system == "Darwin":
            app_dir = self.tb_root / "TB Language.app"
            if app_dir.exists():
                shutil.rmtree(app_dir)
                print("   ✓ Removed app bundle")

        print("   ✓ Uninstall complete!")

get_executable()

Get TB executable path

Source code in toolboxv2/utils/tbx/setup.py

def get_executable(self) -> Path:
    """Get TB executable path"""
    if self.system == "Windows":
        exe = self.tb_root / "bin" / "tb.exe"
    else:
        exe = self.tb_root / "bin" / "tb"

    if not exe.exists():
        # Try target/release
        if self.system == "Windows":
            exe = self.tb_root / "tb-exc" / "target" / "release" / "tb.exe"
        else:
            exe = self.tb_root / "tb-exc" / "target" / "release" / "tb"

    return exe

get_tb_root()

Get toolbox root directory

Source code in toolboxv2/utils/tbx/setup.py

def get_tb_root(self) -> Path:
    """Get toolbox root directory"""
    try:
        from toolboxv2 import tb_root_dir
        return Path(tb_root_dir)
    except ImportError:
        return Path(__file__).parent.parent

setup_all()

Run complete setup

Source code in toolboxv2/utils/tbx/setup.py

def setup_all(self):
    """Run complete setup"""
    print("╔════════════════════════════════════════════════════════════════╗")
    print("║         TB Language - System Integration Setup                 ║")
    print("╚════════════════════════════════════════════════════════════════╝")
    print()

    # Check prerequisites
    if not self.executable.exists():
        print("❌ TB executable not found!")
        print(f"   Expected at: {self.executable}")
        print("   Run 'tb x build' first!")
        return False

    print(f"✓ TB executable found: {self.executable}")
    print()

    # Setup icon
    if not self.setup_icon():
        print("⚠️  Icon setup failed (continuing anyway)")

    # Setup file association
    if self.system == "Windows":
        success = self.setup_windows()
    elif self.system == "Linux":
        success = self.setup_linux()
    elif self.system == "Darwin":
        success = self.setup_macos()
    else:
        print(f"❌ Unsupported system: {self.system}")
        return False

    if success:
        print()
        print("╔════════════════════════════════════════════════════════════════╗")
        print("║                    ✓ Setup Complete!                           ║")
        print("╠════════════════════════════════════════════════════════════════╣")
        print("║  .tbx files are now associated with TB Language                ║")
        print("║  Double-click any .tbx file to run it!                         ║")
        print("╚════════════════════════════════════════════════════════════════╝")

    return success

setup_icon()

Setup icon file

Source code in toolboxv2/utils/tbx/setup.py

    def setup_icon(self) -> bool:
        """Setup icon file"""
        print("📦 Setting up icon...")

        icon_dir = self.tb_root / "resources"
        icon_dir.mkdir(exist_ok=True)

        # Check if icon exists
        if self.icon_path.exists():
            print(f"   ✓ Icon already exists: {self.icon_path}")
            return True

        # Create placeholder icon info
        print(f"   ⚠️  Icon not found at: {self.icon_path}")
        print(f"   📝 Creating placeholder...")

        # Try to create a simple icon reference
        # User needs to provide actual tb_icon.ico file
        placeholder = icon_dir / "README_ICON.txt"
        placeholder.write_text("""
TB Language Icon
================

Place your icon files here:
- tb_icon.ico   (Windows)
- tb_icon.png   (Linux)
- tb_icon.icns  (macOS)

Recommended size: 256x256 px

You can use the ToolBox V2 logo/icon.
        """)

        print(f"   ℹ️  Place icon file at: {self.icon_path}")
        return False

setup_linux()

Setup file association on Linux

Source code in toolboxv2/utils/tbx/setup.py

    def setup_linux(self) -> bool:
        """Setup file association on Linux"""
        print("🐧 Setting up Linux file association...")

        try:
            # Create .desktop file
            desktop_dir = Path.home() / ".local" / "share" / "applications"
            desktop_dir.mkdir(parents=True, exist_ok=True)

            desktop_file = desktop_dir / "tb-language.desktop"

            icon_path = self.icon_path.with_suffix('.png')
            if not icon_path.exists():
                icon_path = "text-x-script"  # Fallback icon

            desktop_content = f"""[Desktop Entry]
Version=1.0
Type=Application
Name=TB Language
Comment=Execute TB Language programs
Exec={self.executable} run %f
Icon={icon_path}
Terminal=false
MimeType=text/x-tb;application/x-tb;
Categories=Development;
"""

            desktop_file.write_text(desktop_content)
            desktop_file.chmod(0o755)
            print(f"   ✓ Created desktop entry: {desktop_file}")

            # Create MIME type
            mime_dir = Path.home() / ".local" / "share" / "mime" / "packages"
            mime_dir.mkdir(parents=True, exist_ok=True)

            mime_file = mime_dir / "tb-language.xml"
            mime_content = """<?xml version="1.0" encoding="UTF-8"?>
<mime-info xmlns="http://www.freedesktop.org/standards/shared-mime-info">
    <mime-type type="text/x-tb">
        <comment>TB Language Program</comment>
        <glob pattern="*.tbx"/>
        <sub-class-of type="text/plain"/>
    </mime-type>
</mime-info>
"""

            mime_file.write_text(mime_content)
            print(f"   ✓ Created MIME type: {mime_file}")

            # Update MIME database
            print("   Updating MIME database...")
            try:
                subprocess.run(["update-mime-database",
                                str(Path.home() / ".local" / "share" / "mime")],
                               check=True, capture_output=True)
                print("   ✓ MIME database updated")
            except:
                print("   ⚠️  Could not update MIME database automatically")
                print("   Run: update-mime-database ~/.local/share/mime")

            # Update desktop database
            print("   Updating desktop database...")
            try:
                subprocess.run(["update-desktop-database", str(desktop_dir)],
                               check=True, capture_output=True)
                print("   ✓ Desktop database updated")
            except:
                print("   ⚠️  Could not update desktop database automatically")

            # Set default application
            try:
                subprocess.run([
                    "xdg-mime", "default", "tb-language.desktop", "text/x-tb"
                ], check=True, capture_output=True)
                print("   ✓ Set as default application for .tbx files")
            except:
                print("   ⚠️  Could not set as default application")

            print("   ✓ Linux setup complete!")
            return True

        except Exception as e:
            print(f"   ❌ Error: {e}")
            return False

setup_macos()

Setup file association on macOS

Source code in toolboxv2/utils/tbx/setup.py

    def setup_macos(self) -> bool:
        """Setup file association on macOS"""
        print("🍎 Setting up macOS file association...")

        try:
            # Create Info.plist for file association
            app_dir = self.tb_root / "TB Language.app"
            contents_dir = app_dir / "Contents"
            macos_dir = contents_dir / "MacOS"
            resources_dir = contents_dir / "Resources"

            # Create directories
            macos_dir.mkdir(parents=True, exist_ok=True)
            resources_dir.mkdir(parents=True, exist_ok=True)

            # Copy executable
            app_executable = macos_dir / "tb"
            if not app_executable.exists():
                shutil.copy(self.executable, app_executable)
                app_executable.chmod(0o755)

            # Create launcher script
            launcher = macos_dir / "TB Language"
            launcher.write_text(f"""#!/bin/bash
if [ "$#" -gt 0 ]; then
    "{app_executable}" run "$@"
else
    "{app_executable}" repl
fi
""")
            launcher.chmod(0o755)

            # Create Info.plist
            plist_file = contents_dir / "Info.plist"
            plist_content = f"""<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
    <key>CFBundleExecutable</key>
    <string>TB Language</string>
    <key>CFBundleIconFile</key>
    <string>tb_icon</string>
    <key>CFBundleIdentifier</key>
    <string>dev.tblang.tb</string>
    <key>CFBundleName</key>
    <string>TB Language</string>
    <key>CFBundlePackageType</key>
    <string>APPL</string>
    <key>CFBundleShortVersionString</key>
    <string>1.0.0</string>
    <key>CFBundleVersion</key>
    <string>1.0.0</string>
    <key>CFBundleDocumentTypes</key>
    <array>
        <dict>
            <key>CFBundleTypeExtensions</key>
            <array>
                <string>tbx</string>
            </array>
            <key>CFBundleTypeIconFile</key>
            <string>tb_icon</string>
            <key>CFBundleTypeName</key>
            <string>TB Language Program</string>
            <key>CFBundleTypeRole</key>
            <string>Editor</string>
            <key>LSHandlerRank</key>
            <string>Owner</string>
        </dict>
    </array>
</dict>
</plist>
"""
            plist_file.write_text(plist_content)
            print(f"   ✓ Created app bundle: {app_dir}")

            # Copy icon if exists
            icon_src = self.icon_path.with_suffix('.icns')
            if icon_src.exists():
                shutil.copy(icon_src, resources_dir / "tb_icon.icns")
                print(f"   ✓ Copied icon")

            # Register with Launch Services
            print("   Registering with Launch Services...")
            try:
                subprocess.run([
                    "/System/Library/Frameworks/CoreServices.framework/Frameworks/LaunchServices.framework/Support/lsregister",
                    "-f", str(app_dir)
                ], check=True, capture_output=True)
                print("   ✓ Registered with Launch Services")
            except:
                print("   ⚠️  Could not register automatically")
                print(f"   Run: open '{app_dir}'")

            print("   ✓ macOS setup complete!")
            return True

        except Exception as e:
            print(f"   ❌ Error: {e}")
            return False

setup_windows()

Setup file association on Windows

Source code in toolboxv2/utils/tbx/setup.py

def setup_windows(self) -> bool:
    """Setup file association on Windows"""
    print("🪟 Setting up Windows file association...")

    try:
        import winreg

        # Create .tbx extension key
        print("   Creating registry entries...")

        # HKEY_CURRENT_USER\Software\Classes\.tbx
        with winreg.CreateKey(winreg.HKEY_CURRENT_USER, r"Software\Classes\.tbx") as key:
            winreg.SetValue(key, "", winreg.REG_SZ, "TBLanguageFile")
            print("   ✓ Registered .tbx extension")

        # HKEY_CURRENT_USER\Software\Classes\TBLanguageFile
        with winreg.CreateKey(winreg.HKEY_CURRENT_USER, r"Software\Classes\TBLanguageFile") as key:
            winreg.SetValue(key, "", winreg.REG_SZ, "TB Language Program")

            # Set icon
            if self.icon_path.exists():
                icon_key = winreg.CreateKey(key, "DefaultIcon")
                winreg.SetValue(icon_key, "", winreg.REG_SZ, str(self.icon_path))
                print(f"   ✓ Set icon: {self.icon_path}")

            # Set open command
            command_key = winreg.CreateKey(key, r"shell\open\command")
            cmd = f'"{self.executable}" run "%1"'
            winreg.SetValue(command_key, "", winreg.REG_SZ, cmd)
            print(f"   ✓ Set open command: {cmd}")

            # Add "Run in Terminal" context menu
            terminal_key = winreg.CreateKey(key, r"shell\run_terminal\command")
            terminal_cmd = f'cmd /k "{self.executable}" run "%1" && pause'
            winreg.SetValue(terminal_key, "", winreg.REG_SZ, terminal_cmd)
            winreg.SetValue(winreg.CreateKey(key, r"shell\run_terminal"), "", winreg.REG_SZ, "Run in Terminal")
            print(f"   ✓ Added 'Run in Terminal' context menu")

            # Add "Edit" context menu
            edit_key = winreg.CreateKey(key, r"shell\edit\command")
            winreg.SetValue(edit_key, "", winreg.REG_SZ, 'notepad "%1"')
            winreg.SetValue(winreg.CreateKey(key, r"shell\edit"), "", winreg.REG_SZ, "Edit")
            print(f"   ✓ Added 'Edit' context menu")

        # Refresh shell
        print("   Refreshing Explorer...")
        try:
            import ctypes
            ctypes.windll.shell32.SHChangeNotify(0x08000000, 0x0000, None, None)
        except:
            print("   ⚠️  Could not refresh Explorer (restart may be needed)")

        print("   ✓ Windows setup complete!")
        return True

    except ImportError:
        print("   ❌ winreg module not available")
        return False
    except Exception as e:
        print(f"   ❌ Error: {e}")
        return False

uninstall()

Remove file associations

Source code in toolboxv2/utils/tbx/setup.py

def uninstall(self):
    """Remove file associations"""
    print("🗑️  Uninstalling file associations...")

    if self.system == "Windows":
        try:
            import winreg
            winreg.DeleteKey(winreg.HKEY_CURRENT_USER, r"Software\Classes\.tbx")
            winreg.DeleteKey(winreg.HKEY_CURRENT_USER, r"Software\Classes\TBLanguageFile")
            print("   ✓ Windows registry cleaned")
        except:
            print("   ⚠️  Could not clean registry")

    elif self.system == "Linux":
        desktop_file = Path.home() / ".local" / "share" / "applications" / "tb-language.desktop"
        mime_file = Path.home() / ".local" / "share" / "mime" / "packages" / "tb-language.xml"

        if desktop_file.exists():
            desktop_file.unlink()
            print("   ✓ Removed desktop entry")

        if mime_file.exists():
            mime_file.unlink()
            print("   ✓ Removed MIME type")

    elif self.system == "Darwin":
        app_dir = self.tb_root / "TB Language.app"
        if app_dir.exists():
            shutil.rmtree(app_dir)
            print("   ✓ Removed app bundle")

    print("   ✓ Uninstall complete!")

main()

Main entry point

Source code in toolboxv2/utils/tbx/setup.py

def main():
    """Main entry point"""
    import argparse

    parser = argparse.ArgumentParser(
        description="TB Language System Integration Setup"
    )
    parser.add_argument('action', choices=['install', 'uninstall'],
                        help='Action to perform')

    args = parser.parse_args()

    setup = TBxSetup()

    if args.action == 'install':
        success = setup.setup_all()
        sys.exit(0 if success else 1)
    elif args.action == 'uninstall':
        setup.uninstall()
        sys.exit(0)

test_tb_lang

TB Language Comprehensive Test Suite Tests all features of the TB language implementation.

Usage

python test_tb_lang.py python test_tb_lang.py --verbose python test_tb_lang.py --filter "test_arithmetic"

assert_contains(code, substring, mode='jit')

Assert that output contains substring.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

def assert_contains(code: str, substring: str, mode: str = "jit"):
    """Assert that output contains substring."""
    success, stdout, stderr = run_tb(code, mode)

    if not success:
        raise AssertionError(f"Execution failed:\n{stderr}")

    if substring not in stdout:
        raise AssertionError(
            f"Output does not contain '{substring}':\n{stdout}"
        )

assert_output(code, expected, mode='jit')

Assert that TB code produces expected output.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

def assert_output(code: str, expected: str, mode: str = "jit"):
    """Assert that TB code produces expected output."""

    success, stdout, stderr = run_tb(code, mode)
    if VERBOSE and not success:
        print(f"code: {code}")
        print()
        print(f"stdout: {stdout}")
    if not success:
        raise AssertionError(f"Execution failed:\n{stderr}")

    actual = stdout.strip()
    expected = expected.strip()

    if actual != expected:
        raise AssertionError(
            f"Output mismatch:\n"
            f"Expected: {repr(expected)}\n"
            f"Got:      {repr(actual)}"
        )

assert_success(code, mode='jit')

Assert that TB code runs without error.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

def assert_success(code: str, mode: str = "jit"):
    """Assert that TB code runs without error."""
    success, stdout, stderr = run_tb(code, mode)
    if VERBOSE and not success:
        print(f"code: {code}")
        print()
    if VERBOSE:
        print(f"stdout: {stdout}")
        print(f"stderr: {stderr}")

    if not success:
        raise AssertionError(f"Execution failed:\n{stderr}")

find_tb_binary()

Find TB binary, trying multiple paths with system-specific extensions.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

def find_tb_binary() -> str:
    """Find TB binary, trying multiple paths with system-specific extensions."""
    from toolboxv2 import tb_root_dir
    # Path to TB binary (relative or absolute)
    TB_BINARY_PATH = str(tb_root_dir / "bin" / "tbx")

    # Alternative paths to try if main path doesn't exist
    ALTERNATIVE_PATHS = [
        str(tb_root_dir / "tb-exc" / "target" / "debug" / "tbx"),
        str(tb_root_dir / "tb-exc" / "target" / "release" / "tbx"),
        #"tbx",  # System PATH
    ]

    # Add system-specific extension
    def add_extension(path: str) -> str:
        if system() == "Windows" and not path.endswith(".exe"):
            return f"{path}.exe"
        return path

    # Prepare paths with proper extensions
    paths_to_try = [add_extension(TB_BINARY_PATH)]
    for alt_path in ALTERNATIVE_PATHS:
        paths_to_try.append(add_extension(alt_path))

    for path in paths_to_try:
        # Check if file exists directly
        if os.path.exists(path):
            return path

        # Use shutil.which for system PATH lookup (cross-platform)
        if shutil.which(path):
            return path

    print(f"{Colors.RED}✗ TB binary not found!{Colors.RESET}")
    print(f"{Colors.YELLOW}Tried paths:{Colors.RESET}")
    for path in paths_to_try:
        print(f"  • {path}")
    print(f"\n{Colors.CYAN}Build the binary with:{Colors.RESET}")
    print(f"  tb run build")
    return ""

run_tb(code, mode='jit', timeout=10)

Run TB code and return (success, stdout, stderr).

Parameters:

Name	Type	Description	Default
code	str	TB source code	required
mode	str	Execution mode (jit, compiled, streaming)	'jit'
timeout	int	Timeout in seconds	10

Returns:

Type	Description
Tuple[bool, str, str]	(success, stdout, stderr)

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

def run_tb(code: str, mode: str = "jit", timeout: int = 10) -> Tuple[bool, str, str]:
    """
    Run TB code and return (success, stdout, stderr).

    Args:
        code: TB source code
        mode: Execution mode (jit, compiled, streaming)
        timeout: Timeout in seconds

    Returns:
        (success, stdout, stderr)
    """
    if mode == "compiled":
        with tempfile.NamedTemporaryFile(suffix='', delete=False) as f:
            output_path = f.name

        try:
            start = time.perf_counter()
            success, stderr = run_tb_compile(code, output_path)
            duration = time.perf_counter() - start
            print(f" -- Compile time ({duration:.3f}s)")
            if not success:
                raise AssertionError(f"Compilation failed:\n{stderr}")

            # Check binary exists
            if not os.path.exists(output_path):
                raise AssertionError("Compiled binary not found")

            # Check binary is executable
            if not os.access(output_path, os.X_OK):
                os.chmod(output_path, 0o755)

            start = time.perf_counter()
            # Run compiled binary
            result = subprocess.run([output_path], capture_output=True, text=True, timeout=timeout//2,
                                    encoding=sys.stdout.encoding or 'utf-8')
            duration = time.perf_counter() - start
            print(f" -- Exec time ({duration:.3f}s)")

            if result.returncode != 0:
                raise AssertionError(f"Compiled binary failed: {result.stderr}")

            return success, result.stdout, result.stderr
        finally:
            if os.path.exists(output_path):
                os.remove(output_path)

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, encoding=sys.stdout.encoding or 'utf-8') as f:
        f.write(code)
        temp_file = f.name

    try:
        result = subprocess.run(
            [TB_BINARY, "run", temp_file, "--mode", mode],
            capture_output=True,
            text=True,
            timeout=timeout,
            encoding=sys.stdout.encoding or 'utf-8',
            errors='replace'
        )

        success = result.returncode == 0
        return success, result.stdout, result.stderr

    except subprocess.TimeoutExpired:
        return False, "", f"Timeout after {timeout}s"

    finally:
        try:
            os.unlink(temp_file)
        except:
            pass

run_tb_compile(code, output_path, target=None)

Compile TB code to binary.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

def run_tb_compile(code: str, output_path: str, target: str = None) -> Tuple[bool, str]:
    """Compile TB code to binary."""
    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, encoding=sys.stdout.encoding or 'utf-8') as f:
        f.write(code)
        temp_file = f.name

    try:
        cmd = [TB_BINARY, "compile", temp_file, output_path]
        if target:
            cmd.extend(["--target", target])

        result = subprocess.run(cmd, capture_output=not VERBOSE, text=True, timeout=60, encoding=sys.stdout.encoding or 'utf-8')

        success = result.returncode == 0
        return success, result.stderr

    finally:
        try:
            os.unlink(temp_file)
        except:
            pass

test(name, category='General')

Decorator for test functions.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

def test(name: str, category: str = "General"):
    """Decorator for test functions."""

    def decorator(func):
        def wrapper():
            # Check filter
            if FILTER and FILTER.lower() not in name.lower():
                return

            # Print test header
            if suite.current_category != category:
                print(f"\n{Colors.BOLD}{Colors.CYAN}[{category}]{Colors.RESET}")
                suite.current_category = category

            print(f"  {Colors.GRAY}Testing:{Colors.RESET} {name}", end=" ", flush=True)

            start = time.perf_counter()
            try:
                func()
                duration = time.perf_counter() - start

                print(f" -- {Colors.GREEN}✓{Colors.RESET} ({duration:.3f}s)")

                suite.add_result(TestResult(
                    name=name,
                    passed=True,
                    duration_ms=duration
                ))

            except AssertionError as e:
                duration = time.perf_counter() - start

                print(f"{Colors.RED}✗{Colors.RESET} ({duration:.0f}s)")
                if VERBOSE:
                    print(f"    {Colors.RED}Error: {str(e)}{Colors.RESET}")

                suite.add_result(TestResult(
                    name=name,
                    passed=False,
                    duration_ms=duration,
                    error_message=str(e)
                ))

            except Exception as e:
                import traceback
                print(traceback.format_exc())
                duration = (time.perf_counter() - start) * 1000

                print(f"{Colors.RED}✗ (Exception){Colors.RESET}")
                if VERBOSE:
                    print(f"    {Colors.RED}{type(e).__name__}: {str(e)}{Colors.RESET}")

                suite.add_result(TestResult(
                    name=name,
                    passed=False,
                    duration_ms=duration,
                    error_message=f"{type(e).__name__}: {str(e)}"
                ))

        return wrapper

    return decorator

test_compiled_arithmetic()

Test basic arithmetic in compiled mode

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Compiled - Basic Arithmetic", "Compiled Modes")
def test_compiled_arithmetic():
    """Test basic arithmetic in compiled mode"""
    code = '''
let x = 10
let y = 20
let sum = x + y
let product = x * y
echo sum
echo product
'''
    success, stdout, stderr = run_tb(code, mode="compiled")
    assert success, f"Compilation failed: {stderr}"
    assert "30" in stdout
    assert "200" in stdout

test_compiled_functions()

Test function definitions in compiled mode

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Compiled - Functions", "Compiled Modes")
def test_compiled_functions():
    """Test function definitions in compiled mode"""
    code = '''
fn fibonacci(n: int) -> int {
    if n <= 1 {
        n
    } else {
        fibonacci(n - 1) + fibonacci(n - 2)
    }
}

let result = fibonacci(10)
echo result
'''
    assert_output(code, "55", mode="compiled")

test_compiled_functions_no_compiled_()

Test function definitions in compiled mode

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Compiled - Functions", "Compiled Modes NO Compiled")
def test_compiled_functions_no_compiled_():
    """Test function definitions in compiled mode"""
    code = '''
fn fibonacci(n: int) -> int {
    if n <= 1 {
        n
    } else {
        fibonacci(n - 1) + fibonacci(n - 2)
    }
}

let result = fibonacci(10)
echo result
'''
    assert_output(code, "55", mode="jit")

test_compiled_loops()

Test loops in compiled mode

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Compiled - Loops", "Compiled Modes")
def test_compiled_loops():
    """Test loops in compiled mode"""
    code = '''
let sum = 0
for i in [1, 2, 3, 4, 5] {
    sum = sum + i
}
echo sum
'''
    assert_output(code, "15", mode="compiled")

test_compiled_parallel()

Test parallel execution in compiled mode

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Compiled - Parallel Execution", "Compiled Modes")
def test_compiled_parallel():
    """Test parallel execution in compiled mode"""
    code = '''
@config {
    runtime_mode: "parallel"
}

let results = parallel {
    10 + 5,
    20 * 2,
    30 - 10
}
for result in results {
    echo result
}
'''
    success, stdout, stderr = run_tb(code, mode="compiled")
    assert success, f"Compilation failed: {stderr}"
    # Results may be in any order due to parallelism
    assert "15" in stdout, f"result {stdout}"
    assert "40" in stdout, f"result {stdout}"
    assert "20" in stdout, f"result {stdout}"

test_compiled_parallel_with_imports()

Test parallel execution with imported functions

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Compiled - Parallel with Imports", "Compiled Modes")
def test_compiled_parallel_with_imports():
    """Test parallel execution with imported functions"""
    lib_code = '''
fn compute_square(x: int) {
    x * x
}

fn compute_cube(x: int) {
    x * x * x
}
'''

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as lib_file:
        lib_file.write(lib_code)
        lib_path = os.path.basename(lib_file.name)

    try:
        main_code = f'''
@config {{
    runtime_mode: "parallel"
}}

@imports {{
    "{lib_path}"
}}

let results = parallel {{
    compute_square(5),
    compute_cube(3),
    compute_square(10)
}}

for result in results {{
    echo result
}}
'''
        success, stdout, stderr = run_tb(main_code, mode="compiled")
        assert success, f"Compilation failed: {stderr}"
        assert "25" in stdout
        assert "27" in stdout
        assert "100" in stdout
    finally:
        os.unlink(lib_path)

test_go_basic_execution()

Test that Go code can be executed.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Go Language Bridge - Basic Execution", "Dependencies - Go Bridge")
def test_go_basic_execution():
    """Test that Go code can be executed."""
    code = '''
let result = go("""
import "fmt"
fmt.Println("Hello from Go!")
""")
'''
    assert_success(code)

test_go_bridge_compiled_mode()

Test that the Go language bridge works in compiled mode.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Go Bridge - Compiled Mode", "Dependencies - Go Bridge")
def test_go_bridge_compiled_mode():
    """Test that the Go language bridge works in compiled mode."""
    code = '''
@config {
    mode: "compiled"
}

let result = go("""
import "fmt"
fmt.Println("Hello from Go in compiled mode!")
""")

echo "Go execution finished."
'''
    assert_contains(code, "Go execution finished.", mode="compiled")

test_go_import_json()

Test marshalling data to JSON using Go's 'encoding/json' package.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Go Import - JSON Marshalling", "Dependencies - Go Bridge")
def test_go_import_json():
    """Test marshalling data to JSON using Go's 'encoding/json' package."""
    code = '''
# This test assumes TB lists/dicts are converted to Go slices/maps
let user_data = [
    { "name": "Alice", "age": 30 },
    { "name": "Bob", "age": 25 }
]
let result = go("""
import (
    "fmt"
    "encoding/json"
)
// user_data is available from TB, assume it's a []map[string]interface{}
// Marshal returns the JSON encoding of user_data.
jsonData, err := json.Marshal(user_data)
if err != nil {
    fmt.Println("Error:", err)
} else {
    fmt.Println(string(jsonData))
}
""")
'''
    assert_success(code)

test_go_import_math()

Test using the Go 'math' standard library package.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Go Import - Math", "Dependencies - Go Bridge")
def test_go_import_math():
    """Test using the Go 'math' standard library package."""
    code = '''
let result = go("""
import (
    "fmt"
    "math"
)
// Sqrt returns the square root of 64
fmt.Println(math.Sqrt(64))
""")
'''
    assert_success(code)

test_go_import_strings()

Test using the Go 'strings' package with a TB variable.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Go Import - Strings", "Dependencies - Go Bridge")
def test_go_import_strings():
    """Test using the Go 'strings' package with a TB variable."""
    code = '''
let my_string = "Hello, Go!"
let result = go("""
import (
    "fmt"
    "strings"
)
// my_string is available from TB context
// ToUpper returns a copy of the string with all letters mapped to their upper case.
fmt.Println(strings.ToUpper(my_string))
""")
'''
    assert_success(code)

test_go_multilang_python_to_go()

Test data flow from a Python block to a Go block.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Go Multi-Language - Python to Go", "Dependencies - Go Bridge")
def test_go_multilang_python_to_go():
    """Test data flow from a Python block to a Go block."""
    code = '''
# Python creates a string
let data_from_py = python("""
print("Data generated by Python")
""")

# Go processes the string from the TB variable
let go_result = go("""
import "fmt"
// data_from_py is available from TB context
// Printf formats according to a format specifier and writes to standard output.
fmt.Printf("Go received: '%s'\\n", data_from_py)
""")
'''
    assert_success(code)

test_import_cache_invalidation()

Test that cache is invalidated when source changes

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Compiled - Cache Invalidation", "Import Compiled Modes")
def test_import_cache_invalidation():
    """Test that cache is invalidated when source changes"""
    print("→ Test: Cache Invalidation")

    lib_path = 'test_cache_lib.tbx'

    # Write initial version
    with open(lib_path, 'w') as f:
        f.write('''
@config {
    mode: "compiled"
}

fn get_value() {
    42
}
''')

    try:
        main_code = f'''
@imports {{
    "{lib_path}"
}}

echo get_value()
'''

        # First run
        success1, stdout1, _ = run_tb(main_code)
        assert success1
        assert "42" in stdout1

        # Modify library
        time.sleep(0.1)  # Ensure timestamp difference
        with open(lib_path, 'w') as f:
            f.write('''
@config {
    mode: "compiled"
}

fn get_value() {
    100
}
''')

        # Second run should use updated version
        success2, stdout2, _ = run_tb(main_code)
        assert success2
        assert "100" in stdout2, f"Cache not invalidated! Got: {stdout2}"

        print("  ✓ Cache invalidation works")

    finally:
        os.unlink(lib_path)

test_import_compiled_caching()

Test that compiled imports are cached

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Compiled - Import Caching", "Import Compiled Modes")
def test_import_compiled_caching():
    """Test that compiled imports are cached"""
    print("→ Test: Compiled Import Caching")

    # Create library with compiled mode
    lib_code = '''
@config {
    mode: "compiled"
}

fn factorial(n: int) {
    if n <= 1 {
        1
    } else {
        n * factorial(n - 1)
    }
}
'''

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as lib_file:
        lib_file.write(lib_code)
        lib_path = os.path.basename(lib_file.name)

    try:
        main_code = f'''

@config {{
    mode: "compiled"
}}

@imports {{
    "{lib_path}"
}}

let result = factorial(5)
echo $result
'''

        # First run - should compile
        start_time = time.perf_counter()
        success1, stdout1, stderr1 = run_tb(main_code)
        first_run_time = time.perf_counter() - start_time

        assert success1, f"First execution failed: {stderr1}"
        assert "120" in stdout1, f"Expected 120, got: {stdout1}"

        # Second run - should use cache
        start_time = time.perf_counter()
        success2, stdout2, stderr2 = run_tb(main_code)
        second_run_time = time.perf_counter() - start_time

        assert success2, f"Second execution failed: {stderr2}"
        assert "120" in stdout2, f"Expected 120, got: {stdout2}"

        # Second run should be significantly faster (using cache)
        assert second_run_time <= first_run_time, \
            f"Cache not used? First: {first_run_time:.2f}s, Second: {second_run_time:.2f}s"

        print(f"  ✓ Caching works (first: {first_run_time:.2f}s, second: {second_run_time:.2f}s)")

    finally:
        os.unlink(lib_path)

test_import_compiled_mode()

Test that imports work in compiled mode

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Import - Compiled Mode", "Import System")
def test_import_compiled_mode():
    """Test that imports work in compiled mode"""
    lib_code = '''
@config {
    mode: "compiled"
    target: "library"
}

fn double(x: int) {
    x * 2
}
'''

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as lib_file:
        lib_file.write(lib_code)
        lib_path = os.path.basename(lib_file.name)

    try:
        main_code = f'''
@config {{
    mode: "compiled"
}}

@imports {{
    "{lib_path}"
}}

let result = double(21)
echo result
'''
        assert_output(main_code, "42", mode="compiled")
    finally:
        os.unlink(lib_path)

test_import_dependency_chain()

Test imports that depend on other imports

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Compiled - Import Dependency Chain", "Import Compiled Modes")
def test_import_dependency_chain():
    """Test imports that depend on other imports"""
    print("→ Test: Import Dependency Chain")

    # Base library
    base_lib = '''
fn double(x: int) {
    x * 2
}
'''

    # Mid library (uses base)
    mid_lib = '''
@imports {
    "base.tbx"
}

fn quadruple(x: int) {
    double(double(x))
}
'''

    # Create base.tbx
    with open('base.tbx', 'w') as f:
        f.write(base_lib)

    # Create mid.tbx
    with open('mid.tbx', 'w') as f:
        f.write(mid_lib)

    try:
        main_code = '''
@imports {
    "mid.tbx"
}

echo quadruple(5)
'''

        success, stdout, stderr = run_tb(main_code)

        assert success, f"Execution failed: {stderr}"
        assert "20" in stdout, f"Expected 20, got: {stdout}"

        print("  ✓ Transitive imports work")

    finally:
        os.unlink('base.tbx')
        os.unlink('mid.tbx')

test_import_jit_vs_compiled()

Test that JIT imports don't get compiled

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Compiled - JIT vs Compiled Imports", "Import Compiled Modes")
def test_import_jit_vs_compiled():
    """Test that JIT imports don't get compiled"""
    print("→ Test: JIT vs Compiled Imports")

    # JIT library
    jit_lib = '''
@config {
    mode: "jit"
}

fn square(x: int) {
    x * x
}
'''

    # Compiled library
    compiled_lib = '''
@config {
    mode: "compiled"
}

fn cube(x: int) {
    x * x * x
}
'''

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as jit_file, \
         tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as compiled_file:

        jit_file.write(jit_lib)
        compiled_file.write(compiled_lib)
        jit_path = os.path.basename(jit_file.name)
        compiled_path = os.path.basename(compiled_file.name)

    try:
        main_code = f'''
@imports {{
    "{jit_path}"
    "{compiled_path}"
}}

echo square(4)
echo cube(3)
'''

        success, stdout, stderr = run_tb(main_code)

        assert success, f"Execution failed: {stderr}"
        assert "16" in stdout, f"Expected 16 from square(4), got: {stdout}"
        assert "27" in stdout, f"Expected 27 from cube(3), got: {stdout}"

        print("  ✓ Mixed JIT/Compiled imports work")

    finally:
        os.unlink(jit_path)
        os.unlink(compiled_path)

test_import_missing_file()

Test error handling for missing import files

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Import - Missing File Error", "Import Errors")
def test_import_missing_file():
    """Test error handling for missing import files"""
    code = '''
@imports {
    "nonexistent_file.tbx"
}

echo "This should not run"
'''
    success, stdout, stderr = run_tb(code)
    assert not success, "Should fail for missing import file"
    assert "not found" in stderr.lower() or "import" in stderr.lower()

test_import_multiple_files()

Test importing multiple .tbx files

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Import - Multiple Files", "Import System")
def test_import_multiple_files():
    """Test importing multiple .tbx files"""
    # Create math library
    math_lib = '''
fn add(x, y) {
    x + y
}
'''

    # Create string library
    string_lib = '''
fn greet(name: string) {
    echo "Hello,"name "!"
}
'''

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as math_file, \
        tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as string_file:

        math_file.write(math_lib)
        string_file.write(string_lib)
        math_path = os.path.basename(math_file.name)
        string_path = os.path.basename(string_file.name)

    try:
        main_code = f'''
@imports {{
    "{math_path}"
    "{string_path}"
}}

let sum = add(3, 7)
echo sum
greet("World")
'''
        success, stdout, stderr = run_tb(main_code)
        assert success, f"Execution failed: {stderr}"
        assert "10" in stdout, f"Expected '10' in output, got: {stdout}"
        assert "Hello, World !" in stdout, f"Expected Hello, World! in output, got: {stdout}"
    finally:
        os.unlink(math_path)
        os.unlink(string_path)

test_import_no_circular()

Test that circular imports are handled (imports are not recursive)

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Import - Circular Import Protection", "Import Errors")
def test_import_no_circular():
    """Test that circular imports are handled (imports are not recursive)"""
    # Create file that tries to import itself
    lib_code = '''
fn test_func() {
    echo "Hello"
}
'''

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as lib_file:
        lib_file.write(lib_code)
        lib_path = os.path.basename(lib_file.name)

    try:
        # This should work because imports are not recursive
        main_code = f'''
@imports {{
    "{lib_path}"
}}

test_func()
'''
        assert_success(main_code)
    finally:
        os.unlink(lib_path)

test_import_relative_paths()

Test importing with relative directory paths

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Import - Relative Paths", "Import System")
def test_import_relative_paths():
    """Test importing with relative directory paths"""
    # Create subdirectory
    lib_dir = tempfile.mkdtemp(dir='..')
    lib_dir_name = os.path.basename(lib_dir)

    try:
        # Create library in subdirectory
        lib_code = '''
fn multiply(x: int, y: int) {
    x * y
}
'''
        lib_path = os.path.join(lib_dir, 'math.tbx')
        with open(lib_path, 'w') as f:
            f.write(lib_code)

        main_code = f'''
@imports {{
    "{lib_dir_name}/math.tbx"
}}

let result = multiply(6, 7)
echo result
'''
        assert_output(main_code, "42")
    finally:
        shutil.rmtree(lib_dir, ignore_errors=True)

test_import_single_file()

Test importing a single .tbx file

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Import - Single File", "Import System")
def test_import_single_file():
    """Test importing a single .tbx file"""
    # Create library file
    lib_code = '''
fn double(x: int) {
    x * 2
}

fn triple(x: int) {
    x * 3
}
'''

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as lib_file:
        lib_file.write(lib_code)
        lib_path = os.path.basename(lib_file.name)

    try:
        # Create main file that imports library
        main_code = f'''
@imports {{
    "{lib_path}"
}}

let result = double(5)
echo result
'''
        assert_output(main_code, "10")
    finally:
        os.unlink(lib_path)

test_import_with_variables()

Test importing file with global variables

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Import - With Variables", "Import System")
def test_import_with_variables():
    """Test importing file with global variables"""
    lib_code = '''
let PI = 3.14159
let E = 2.71828

fn circle_area(radius: float) {
    PI * radius * radius
}
'''

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as lib_file:
        lib_file.write(lib_code)
        lib_path = os.path.basename(lib_file.name)

    try:
        main_code = f'''
@imports {{
    "{lib_path}"
}}

echo PI
let area = circle_area(2.0)
echo area
'''
        success, stdout, stderr = run_tb(main_code)
        assert success, f"Execution failed: {stderr}"
        assert "3.14159" in stdout or "3.14" in stdout
    finally:
        os.unlink(lib_path)

test_mixed_full_stack()

Comprehensive test with all features combined

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Mixed - Full Stack Test", "Mixed Features")
def test_mixed_full_stack():
    """Comprehensive test with all features combined"""
    helpers_lib = '''
fn compute(x: int) {
    x * 2
}

fn parallel_sum(numbers: list) {
    let sum = 0
    for n in numbers {
        sum = sum + n
    }
    sum
}
'''

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as lib_file:
        lib_file.write(helpers_lib)
        lib_path = os.path.basename(lib_file.name)

    try:
        main_code = f'''
@config {{
    mode: "jit"
}}

@shared {{
    total: 0
}}

@imports {{
    "{lib_path}"
}}

let value = compute(21)
total = parallel_sum([1, 2, 3, 4, 5])
echo value
echo total
'''
        success, stdout, stderr = run_tb(main_code, mode="jit")
        assert success, f"Execution failed: {stderr}"
        assert "42" in stdout
        assert "15" in stdout
    finally:
        os.unlink(lib_path)

test_mixed_import_async_language()

Test imports with async and language bridges

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Mixed - Import  + Language Bridge", "Mixed Features")
def test_mixed_import_async_language():
    """Test imports with async and language bridges"""
    lib_code = '''
fn get_data() {

    python("import sys; print(sys.version.split()[0])")

}
'''

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as lib_file:
        lib_file.write(lib_code)
        lib_path = os.path.basename(lib_file.name)

    try:
        main_code = f'''

@imports {{
    "{lib_path}"
}}

let version = get_data()
echo "Python version detected"
'''
        assert_contains(main_code, "Python version detected", mode="jit")
    finally:
        os.unlink(lib_path)

test_mixed_multiple_imports_parallel()

Test multiple imports with parallel execution

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Mixed - Multiple Imports + Parallel", "Mixed Features")
def test_mixed_multiple_imports_parallel():
    """Test multiple imports with parallel execution"""
    math_lib = '''
fn factorial(n: int) -> int {
    if n <= 1 {
        1
    } else {
        n * factorial(n - 1)
    }
}
'''

    utils_lib = '''
fn power(base: int, exp: int) -> int {
    if exp == 0 {
        1
    } else {
        base * power(base, exp - 1)
    }
}
'''

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as math_file, \
        tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as utils_file:

        math_file.write(math_lib)
        utils_file.write(utils_lib)
        math_path = os.path.basename(math_file.name)
        utils_path = os.path.basename(utils_file.name)

    try:
        main_code = f'''
@config {{
    runtime_mode: "parallel"
}}

@imports {{
    "{math_path}"
    "{utils_path}"
}}

let results = parallel {{
    factorial(5),
    power(2, 10),
    factorial(6)
}}

for result in results {{
    echo result
}}
'''
        success, stdout, stderr = run_tb(main_code, mode="compiled")
        assert success, f"Compilation failed: {stderr}"
        assert "120" in stdout  # 5!
        assert "1024" in stdout  # 2^10
        assert "720" in stdout  # 6!
    finally:
        os.unlink(math_path)
        os.unlink(utils_path)

test_mixed_shared_imports()

Test shared variables with imports

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Mixed - Shared Variables + Imports", "Mixed Features")
def test_mixed_shared_imports():
    """Test shared variables with imports"""
    lib_code = '''
fn increment_counter() {
    counter + 1
}
'''

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as lib_file:
        lib_file.write(lib_code)
        lib_path = os.path.basename(lib_file.name)

    try:
        main_code = f'''
@shared {{
    counter: 0
}}

@imports {{
    "{lib_path}"
}}

counter = increment_counter()
counter = increment_counter()
echo counter
'''
        assert_output(main_code, "2")
    finally:
        os.unlink(lib_path)

test_mixed_shared_imports_mixed_lang()

Test shared variables with imports

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Mixed - Shared Variables + Imports + Python Bridge", "Mixed Features")
def test_mixed_shared_imports_mixed_lang():
    """Test shared variables with imports"""
    lib_code = '''
fn increment_counter() {python("print(counter + 1, end='')")}

'''

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as lib_file:
        lib_file.write(lib_code)
        lib_path = os.path.basename(lib_file.name)

    try:
        main_code = f'''
@shared {{
    counter: 0
}}

@imports {{
    "{lib_path}"
}}

counter = increment_counter()
counter = increment_counter()
echo counter
'''
        assert_output(main_code, "2")
    finally:
        os.unlink(lib_path)

test_type_annotations()

Test that type annotations are correctly handled.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Type Annotations - Basic Types", "Type Annotations")
def test_type_annotations():
    """Test that type annotations are correctly handled."""
    code = '''@config { mode: "jit", type_system: "static" }

let age: int = python("print(42)")
let price: float = python("print(19.99)")
let name: string = python("""x = "Alice"
x""")
let active: bool = python("print(True)")
let scores: list = python("print([85, 92, 78])")

echo "Age: $age"           // Age: 42
echo "Price: $price"       // Price: 19.99
echo "Name: $name"         // Name: Alice
echo "Active: $active"     // Active: true'''
    assert_output(code, "Age: 42\nPrice: 19.99\nName: Alice\nActive: true")

test_type_annotations_()

Test that type annotations are correctly handled.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Type Annotations - Basic Types", "Type Annotations")
def test_type_annotations_():
    """Test that type annotations are correctly handled."""
    code = '''@config { mode: "jit", type_system: "static" }

let age: int = python("print(42)")
let price: float = python("19.99")
let name: string = python("Alice")
let active: bool = python("True")
let scores: list<int> = python("[85, 92, 78]")

echo "Age: $age"           // Age: 42
echo "Price: $price"       // Price: 19.99
echo "Name: $name"         // Name: Alice
echo "Active: $active"     // Active: true'''
    assert_output(code, "Age: 42\nPrice: 19.99\nName: Alice\nActive: true")

test_type_annotations_auto()

Test that type annotations are correctly handled.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Type Annotations - Auto Type Inference", "Type Annotations")
def test_type_annotations_auto():
    """Test that type annotations are correctly handled."""
    code = '''@config { mode: "jit", type_system: "static" }

let age: int = python("print(42)")
let price: float = python("print(19.99)")
let name: string = python("print('Alice')")
let active: bool = python("print(True)")
let scores: list = python("print([85, 92, 78])")

echo type_of(age)           // Age: 42
echo type_of(price)       // Price: 19.99
echo type_of(name)         // Name: Alice
echo type_of(active)         // Active: true
echo type_of(scores)     // Scores: [85, 92, 78]'''
    assert_output(code, "int\nfloat\nstring\nbool\nlist")

test_type_annotations_auto_compiled()

Test that type annotations are correctly handled.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Type Annotations - Auto Type Inference  - Compiled", "Type Annotations - Compiled")
def test_type_annotations_auto_compiled():
    """Test that type annotations are correctly handled."""
    code = '''@config { mode: "compiled", type_system: "static" }

let age: int = python("print(42)")
let price: float = python("print(19.99)")
let name: string = python("print('Alice')")
let active: bool = python("print(True)")
let scores: list<int> = python("print([85, 92, 78])")

echo type_of($age)           // Age: 42
echo type_of($price)       // Price: 19.99
echo type_of($name)         // Name: Alice
echo type_of(active)         // Active: true
echo type_of(scores)     // Scores: [85, 92, 78]'''
    assert_output(code, "int\nfloat\nstring\nbool\nlist<int>", mode="compiled")

test_type_annotations_compiled()

Test that type annotations are correctly handled.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py

@test("Type Annotations - Basic Type - Compiled", "Type Annotations - Compiled")
def test_type_annotations_compiled():
    """Test that type annotations are correctly handled."""
    code = '''@config { mode: "compiled", type_system: "static" }

let age: int = python("print(42)")
let price: float = python("print(19.99)")
let name: string = python("print('Alice')")
let active: bool = python("print(True)")
let scores: list = python("print([85, 92, 78])")

echo "Age: $age"           // Age: 42
echo "Price: $price"       // Price: 19.99
echo "Name: $name"         // Name: Alice
echo "Active: $active"     // Active: true'''
    assert_output(code, "Age: 42\nPrice: 19.99\nName: Alice\nActive: true", mode="compiled")

toolbox

Main module.

App

Source code in toolboxv2/utils/toolbox.py

class App(AppType, metaclass=Singleton):

    def __init__(self, prefix: str = "", args=AppArgs().default()):
        if "test" not in prefix:
            prefix = "main"
        super().__init__(prefix, args)
        self._web_context = None
        t0 = time.perf_counter()
        abspath = os.path.abspath(__file__)
        self.system_flag = system()  # Linux: Linux Mac: Darwin Windows: Windows

        self.appdata = os.getenv('APPDATA') if os.name == 'nt' else os.getenv('XDG_CONFIG_HOME') or os.path.expanduser(
                '~/.config') if os.name == 'posix' else None

        if self.system_flag == "Darwin" or self.system_flag == "Linux":
            dir_name = os.path.dirname(abspath).replace("/utils", "")
        else:
            dir_name = os.path.dirname(abspath).replace("\\utils", "")

        self.start_dir = str(dir_name)

        self.bg_tasks = []

        lapp = dir_name + '\\.data\\'

        if not prefix:
            if not os.path.exists(f"{lapp}last-app-prefix.txt"):
                os.makedirs(lapp, exist_ok=True)
                open(f"{lapp}last-app-prefix.txt", "a").close()
            with open(f"{lapp}last-app-prefix.txt") as prefix_file:
                cont = prefix_file.read()
                if cont:
                    prefix = cont.rstrip()
        else:
            if not os.path.exists(f"{lapp}last-app-prefix.txt"):
                os.makedirs(lapp, exist_ok=True)
                open(f"{lapp}last-app-prefix.txt", "a").close()
            with open(f"{lapp}last-app-prefix.txt", "w") as prefix_file:
                prefix_file.write(prefix)

        self.prefix = prefix

        node_ = node()

        if 'localhost' in node_ and (host := os.getenv('HOSTNAME', 'localhost')) != 'localhost':
            node_ = node_.replace('localhost', host)
        self.id = prefix + '-' + node_
        self.globals = {
            "root": {**globals()},
        }
        self.locals = {
            "user": {'app': self, **locals()},
        }

        identification = self.id
        collective_identification = self.id
        if "test" in prefix:
            if self.system_flag == "Darwin" or self.system_flag == "Linux":
                start_dir = self.start_dir.replace("ToolBoxV2/toolboxv2", "toolboxv2")
            else:
                start_dir = self.start_dir.replace("ToolBoxV2\\toolboxv2", "toolboxv2")
            self.data_dir = start_dir + '\\.data\\' + "test"
            self.config_dir = start_dir + '\\.config\\' + "test"
            self.info_dir = start_dir + '\\.info\\' + "test"
        elif identification.startswith('collective-'):
            collective_identification = identification.split('-')[1]
            self.data_dir = self.start_dir + '\\.data\\' + collective_identification
            self.config_dir = self.start_dir + '\\.config\\' + collective_identification
            self.info_dir = self.start_dir + '\\.info\\' + collective_identification
            self.id = collective_identification
        else:
            self.data_dir = self.start_dir + '\\.data\\' + identification
            self.config_dir = self.start_dir + '\\.config\\' + identification
            self.info_dir = self.start_dir + '\\.info\\' + identification

        if self.appdata is None:
            self.appdata = self.data_dir
        else:
            self.appdata += "/ToolBoxV2"

        if not os.path.exists(self.appdata):
            os.makedirs(self.appdata, exist_ok=True)
        if not os.path.exists(self.data_dir):
            os.makedirs(self.data_dir, exist_ok=True)
        if not os.path.exists(self.config_dir):
            os.makedirs(self.config_dir, exist_ok=True)
        if not os.path.exists(self.info_dir):
            os.makedirs(self.info_dir, exist_ok=True)

        self.print(f"Starting ToolBox as {prefix} from :", Style.Bold(Style.CYAN(f"{os.getcwd()}")))

        logger_info_str, self.logger, self.logging_filename = self.set_logger(args.debug)

        self.print("Logger " + logger_info_str)
        self.print("================================")
        self.logger.info("Logger initialized")
        get_logger().info(Style.GREEN("Starting Application instance"))
        if args.init and args.init is not None and self.start_dir not in sys.path:
            sys.path.append(self.start_dir)

        __version__ = get_version_from_pyproject()
        self.version = __version__

        self.keys = {
            "MACRO": "macro~~~~:",
            "MACRO_C": "m_color~~:",
            "HELPER": "helper~~~:",
            "debug": "debug~~~~:",
            "id": "name-spa~:",
            "st-load": "mute~load:",
            "comm-his": "comm-his~:",
            "develop-mode": "dev~mode~:",
            "provider::": "provider::",
        }

        defaults = {
            "MACRO": ['Exit'],
            "MACRO_C": {},
            "HELPER": {},
            "debug": args.debug,
            "id": self.id,
            "st-load": False,
            "comm-his": [[]],
            "develop-mode": False,
        }
        self.config_fh = FileHandler(collective_identification + ".config", keys=self.keys, defaults=defaults)
        self.config_fh.load_file_handler()
        self._debug = args.debug
        self.flows = {}
        self.dev_modi = self.config_fh.get_file_handler(self.keys["develop-mode"])
        if self.config_fh.get_file_handler("provider::") is None:
            self.config_fh.add_to_save_file_handler("provider::", "http://localhost:" + str(
                self.args_sto.port) if os.environ.get("HOSTNAME","localhost") == "localhost" else "https://simplecore.app")
        self.functions = {}
        self.modules = {}

        self.interface_type = ToolBoxInterfaces.native
        self.PREFIX = Style.CYAN(f"~{node()}@>")
        self.alive = True
        self.called_exit = False, time.time()

        self.print(f"Infos:\n  {'Name':<8} -> {node()}\n  {'ID':<8} -> {self.id}\n  {'Version':<8} -> {self.version}\n")

        self.logger.info(
            Style.GREEN(
                f"Finish init up in {time.perf_counter() - t0:.2f}s"
            )
        )

        self.args_sto = args
        self.loop = None

        from .system.session import Session
        self.session: Session = Session(self.get_username())
        if len(sys.argv) > 2 and sys.argv[1] == "db":
            return
        from toolboxv2.utils.clis.db_cli_manager import ClusterManager, get_executable_path
        self.cluster_manager = ClusterManager()
        online_list, server_list = self.cluster_manager.status_all(silent=True)
        if not server_list:
            self.cluster_manager.start_all(get_executable_path(), self.version)
            _, server_list = self.cluster_manager.status_all()
        from .extras.blobs import BlobStorage
        self.root_blob_storage = BlobStorage(servers=server_list, storage_directory=self.data_dir+ '\\blob_cache\\')
        self.mkdocs = add_to_app(self)
        # self._start_event_loop()

    def _start_event_loop(self):
        """Starts the asyncio event loop in a separate thread."""
        if self.loop is None:
            self.loop = asyncio.new_event_loop()
            self.loop_thread = threading.Thread(target=self.loop.run_forever, daemon=True)
            self.loop_thread.start()

    def get_username(self, get_input=False, default="loot") -> str:
        user_name = self.config_fh.get_file_handler("ac_user:::")
        if get_input and user_name is None:
            user_name = input("Input your username: ")
            self.config_fh.add_to_save_file_handler("ac_user:::", user_name)
        if user_name is None:
            user_name = default
            self.config_fh.add_to_save_file_handler("ac_user:::", user_name)
        return user_name

    def set_username(self, username):
        return self.config_fh.add_to_save_file_handler("ac_user:::", username)

    @staticmethod
    def exit_main(*args, **kwargs):
        """proxi attr"""

    @staticmethod
    def hide_console(*args, **kwargs):
        """proxi attr"""

    @staticmethod
    def show_console(*args, **kwargs):
        """proxi attr"""

    @staticmethod
    def disconnect(*args, **kwargs):
        """proxi attr"""

    def set_logger(self, debug=False):
        if "test" in self.prefix and not debug:
            logger, logging_filename = setup_logging(logging.NOTSET, name="toolbox-test", interminal=True,
                                                     file_level=logging.NOTSET, app_name=self.id)
            logger_info_str = "in Test Mode"
        elif "live" in self.prefix and not debug:
            logger, logging_filename = setup_logging(logging.DEBUG, name="toolbox-live", interminal=False,
                                                     file_level=logging.WARNING, app_name=self.id)
            logger_info_str = "in Live Mode"
            # setup_logging(logging.WARNING, name="toolbox-live", is_online=True
            #              , online_level=logging.WARNING).info("Logger initialized")
        elif "debug" in self.prefix or self.prefix.endswith("D"):
            self.prefix = self.prefix.replace("-debug", '').replace("debug", '')
            logger, logging_filename = setup_logging(logging.DEBUG, name="toolbox-debug", interminal=True,
                                                     file_level=logging.WARNING, app_name=self.id)
            logger_info_str = "in debug Mode"
            self.debug = True
        elif debug:
            logger, logging_filename = setup_logging(logging.DEBUG, name=f"toolbox-{self.prefix}-debug",
                                                     interminal=True,
                                                     file_level=logging.DEBUG, app_name=self.id)
            logger_info_str = "in args debug Mode"
        else:
            logger, logging_filename = setup_logging(logging.ERROR, name=f"toolbox-{self.prefix}", app_name=self.id)
            logger_info_str = "in Default"

        return logger_info_str, logger, logging_filename

    @property
    def debug(self):
        return self._debug

    @debug.setter
    def debug(self, value):
        if not isinstance(value, bool):
            self.logger.debug(f"Value must be an boolean. is : {value} type of {type(value)}")
            raise ValueError("Value must be an boolean.")

        # self.logger.info(f"Setting debug {value}")
        self._debug = value

    def debug_rains(self, e):
        if self.debug:
            import traceback
            x = "="*5
            x += " DEBUG "
            x += "="*5
            self.print(x)
            self.print(traceback.format_exc())
            self.print(x)
            raise e
        else:
            self.logger.error(f"Error: {e}")
            import traceback
            x = "="*5
            x += " DEBUG "
            x += "="*5
            self.print(x)
            self.print(traceback.format_exc())
            self.print(x)

    def set_flows(self, r):
        self.flows = r

    async def run_flows(self, name, **kwargs):
        from ..flows import flows_dict as flows_dict_func
        if name not in self.flows:
            self.flows = {**self.flows, **flows_dict_func(s=name, remote=True)}
        if name in self.flows:
            if asyncio.iscoroutinefunction(self.flows[name]):
                return await self.flows[name](get_app(from_="runner"), self.args_sto, **kwargs)
            else:
                return self.flows[name](get_app(from_="runner"), self.args_sto, **kwargs)
        else:
            print("Flow not found, active flows:", len(self.flows.keys()))

    def _coppy_mod(self, content, new_mod_dir, mod_name, file_type='py'):

        mode = 'xb'
        self.logger.info(f" coppy mod {mod_name} to {new_mod_dir} size : {sys.getsizeof(content) / 8388608:.3f} mb")

        if not os.path.exists(new_mod_dir):
            os.makedirs(new_mod_dir)
            with open(f"{new_mod_dir}/__init__.py", "w") as nmd:
                nmd.write(f"__version__ = '{self.version}'")

        if os.path.exists(f"{new_mod_dir}/{mod_name}.{file_type}"):
            mode = False

            with open(f"{new_mod_dir}/{mod_name}.{file_type}", 'rb') as d:
                runtime_mod = d.read()  # Testing version but not efficient

            if len(content) != len(runtime_mod):
                mode = 'wb'

        if mode:
            with open(f"{new_mod_dir}/{mod_name}.{file_type}", mode) as f:
                f.write(content)

    def _pre_lib_mod(self, mod_name, path_to="./runtime", file_type='py'):
        working_dir = self.id.replace(".", "_")
        lib_mod_dir = f"toolboxv2.runtime.{working_dir}.mod_lib."

        self.logger.info(f"pre_lib_mod {mod_name} from {lib_mod_dir}")

        postfix = "_dev" if self.dev_modi else ""
        mod_file_dir = f"./mods{postfix}/{mod_name}.{file_type}"
        new_mod_dir = f"{path_to}/{working_dir}/mod_lib"
        with open(mod_file_dir, "rb") as c:
            content = c.read()
        self._coppy_mod(content, new_mod_dir, mod_name, file_type=file_type)
        return lib_mod_dir

    def _copy_load(self, mod_name, file_type='py', **kwargs):
        loc = self._pre_lib_mod(mod_name, file_type)
        return self.inplace_load_instance(mod_name, loc=loc, **kwargs)

    def helper_install_pip_module(self, module_name):
        if 'main' in self.id:
            return
        self.print(f"Installing {module_name} GREEDY")
        os.system(f"{sys.executable} -m pip install {module_name}")

    def python_module_import_classifier(self, mod_name, error_message):

        if error_message.startswith("No module named 'toolboxv2.utils"):
            return Result.default_internal_error(f"404 {error_message.split('utils')[1]} not found")
        if error_message.startswith("No module named 'toolboxv2.mods"):
            if mod_name.startswith('.'):
                return
            return self.run_a_from_sync(self.a_run_any, ("CloudM", "install"), module_name=mod_name)
        if error_message.startswith("No module named '"):
            pip_requ = error_message.split("'")[1].replace("'", "").strip()
            # if 'y' in input(f"\t\t\tAuto install {pip_requ} Y/n").lower:
            return self.helper_install_pip_module(pip_requ)
            # return Result.default_internal_error(f"404 {pip_requ} not found")

    def inplace_load_instance(self, mod_name, loc="toolboxv2.mods.", spec='app', save=True, mfo=None):
        if self.dev_modi and loc == "toolboxv2.mods.":
            loc = "toolboxv2.mods_dev."
        if spec=='app' and self.mod_online(mod_name):
            self.logger.info(f"Reloading mod from : {loc + mod_name}")
            self.remove_mod(mod_name, spec=spec, delete=False)

        if (os.path.exists(self.start_dir + '/mods/' + mod_name) or os.path.exists(
            self.start_dir + '/mods/' + mod_name + '.py')) and (
            os.path.isdir(self.start_dir + '/mods/' + mod_name) or os.path.isfile(
            self.start_dir + '/mods/' + mod_name + '.py')):
            try:
                if mfo is None:
                    modular_file_object = import_module(loc + mod_name)
                else:
                    modular_file_object = mfo
                self.modules[mod_name] = modular_file_object
            except ModuleNotFoundError as e:
                self.logger.error(Style.RED(f"module {loc + mod_name} not found is type sensitive {e}"))
                self.print(Style.RED(f"module {loc + mod_name} not found is type sensitive {e}"))
                if self.debug or self.args_sto.sysPrint:
                    self.python_module_import_classifier(mod_name, str(e))
                self.debug_rains(e)
                return None
        else:
            self.sprint(f"module {loc + mod_name} is not valid")
            return None
        if hasattr(modular_file_object, "Tools"):
            tools_class = modular_file_object.Tools
        else:
            if hasattr(modular_file_object, "name"):
                tools_class = modular_file_object
                modular_file_object = import_module(loc + mod_name)
            else:
                tools_class = None

        modular_id = None
        instance = modular_file_object
        app_instance_type = "file/application"

        if tools_class is None:
            modular_id = modular_file_object.Name if hasattr(modular_file_object, "Name") else mod_name

        if tools_class is None and modular_id is None:
            modular_id = str(modular_file_object.__name__)
            self.logger.warning(f"Unknown instance loaded {mod_name}")
            return modular_file_object

        if tools_class is not None:
            tools_class = self.save_initialized_module(tools_class, spec)
            modular_id = tools_class.name
            app_instance_type = "functions/class"
        else:
            instance.spec = spec
        # if private:
        #     self.functions[modular_id][f"{spec}_private"] = private

        if not save:
            return instance if tools_class is None else tools_class

        return self.save_instance(instance, modular_id, spec, app_instance_type, tools_class=tools_class)

    def save_instance(self, instance, modular_id, spec='app', instance_type="file/application", tools_class=None):

        if modular_id in self.functions and tools_class is None:
            if self.functions[modular_id].get(f"{spec}_instance", None) is None:
                self.functions[modular_id][f"{spec}_instance"] = instance
                self.functions[modular_id][f"{spec}_instance_type"] = instance_type
            else:
                self.print("Firest instance stays use new spec to get new instance")
                if modular_id in self.functions and self.functions[modular_id].get(f"{spec}_instance", None) is not None:
                    return self.functions[modular_id][f"{spec}_instance"]
                else:
                    raise ImportError(f"Module already known {modular_id} and not avalabel reload using other spec then {spec}")

        elif tools_class is not None:
            if modular_id not in self.functions:
                self.functions[modular_id] = {}
            self.functions[modular_id][f"{spec}_instance"] = tools_class
            self.functions[modular_id][f"{spec}_instance_type"] = instance_type

            try:
                if not hasattr(tools_class, 'tools'):
                    tools_class.tools = {"Version": tools_class.get_version, 'name': tools_class.name}
                for function_name in list(tools_class.tools.keys()):
                    t_function_name = function_name.lower()
                    if t_function_name != "all" and t_function_name != "name":
                        self.tb(function_name, mod_name=modular_id)(tools_class.tools.get(function_name))
                self.functions[modular_id][f"{spec}_instance_type"] += "/BC"
                if hasattr(tools_class, 'on_exit'):
                    if "on_exit" in self.functions[modular_id]:
                        self.functions[modular_id]["on_exit"].append(tools_class.on_exit)
                    else:
                        self.functions[modular_id]["on_exit"] = [tools_class.on_exit]
            except Exception as e:
                self.logger.error(f"Starting Module {modular_id} compatibility failed with : {e}")
                pass
        elif modular_id not in self.functions and tools_class is None:
            self.functions[modular_id] = {}
            self.functions[modular_id][f"{spec}_instance"] = instance
            self.functions[modular_id][f"{spec}_instance_type"] = instance_type

        else:
            raise ImportError(f"Modular {modular_id} is not a valid mod")
        on_start = self.functions[modular_id].get("on_start")
        if on_start is not None:
            i = 1
            for f in on_start:
                try:
                    f_, e = self.get_function((modular_id, f), state=True, specification=spec)
                    if e == 0:
                        self.logger.info(Style.GREY(f"Running On start {f} {i}/{len(on_start)}"))
                        if asyncio.iscoroutinefunction(f_):
                            self.print(f"Async on start is only in Tool claas supported for {modular_id}.{f}" if tools_class is None else f"initialization starting soon for {modular_id}.{f}")
                            self.run_bg_task_advanced(f_)
                        else:
                            o = f_()
                            if o is not None:
                                self.print(f"Function {modular_id} On start result: {o}")
                    else:
                        self.logger.warning(f"starting function not found {e}")
                except Exception as e:
                    self.logger.debug(Style.YELLOW(
                        Style.Bold(f"modular:{modular_id}.{f} on_start error {i}/{len(on_start)} -> {e}")))
                    self.debug_rains(e)
                finally:
                    i += 1
        return instance if tools_class is None else tools_class

    def save_initialized_module(self, tools_class, spec):
        tools_class.spec = spec
        live_tools_class = tools_class(app=self)
        return live_tools_class

    def mod_online(self, mod_name, installed=False):
        if installed and mod_name not in self.functions:
            self.save_load(mod_name)
        return mod_name in self.functions

    def _get_function(self,
                      name: Enum or None,
                      state: bool = True,
                      specification: str = "app",
                      metadata=False, as_str: tuple or None = None, r=0, **kwargs):

        if as_str is None and isinstance(name, Enum):
            modular_id = str(name.NAME.value)
            function_id = str(name.value)
        elif as_str is None and isinstance(name, list):
            modular_id, function_id = name[0], name[1]
        else:
            modular_id, function_id = as_str

        self.logger.info(f"getting function : {specification}.{modular_id}.{function_id}")

        if modular_id not in self.functions:
            if r == 0:
                self.save_load(modular_id, spec=specification)
                return self.get_function(name=(modular_id, function_id),
                                         state=state,
                                         specification=specification,
                                         metadata=metadata,
                                         r=1)
            self.logger.warning(f"function modular not found {modular_id} 404")
            return "404", 404

        if function_id not in self.functions[modular_id]:
            self.logger.warning(f"function data not found {modular_id}.{function_id} 404")
            return "404", 404

        function_data = self.functions[modular_id][function_id]

        if isinstance(function_data, list):
            print(f"functions {function_id} : {function_data}")
            function_data = self.functions[modular_id][function_data[kwargs.get('i', -1)]]
            print(f"functions {modular_id} : {function_data}")
        function = function_data.get("func")
        params = function_data.get("params")

        state_ = function_data.get("state")
        if state_ is not None and state != state_:
            state = state_

        if function is None:
            self.logger.warning("No function found")
            return "404", 404

        if params is None:
            self.logger.warning("No function (params) found")
            return "404", 301

        if metadata and not state:
            self.logger.info("returning metadata stateless")
            return (function_data, function), 0

        if not state:  # mens a stateless function
            self.logger.info("returning stateless function")
            return function, 0

        instance = self.functions[modular_id].get(f"{specification}_instance")

        # instance_type = self.functions[modular_id].get(f"{specification}_instance_type", "functions/class")

        if params[0] == 'app':
            instance = get_app(from_=f"fuction {specification}.{modular_id}.{function_id}")

        if instance is None and self.alive:
            self.inplace_load_instance(modular_id, spec=specification)
            instance = self.functions[modular_id].get(f"{specification}_instance")

        if instance is None:
            self.logger.warning("No live Instance found")
            return "404", 400

        # if instance_type.endswith("/BC"):  # for backwards compatibility  functions/class/BC old modules
        #     # returning as stateless
        #     # return "422", -1
        #     self.logger.info(
        #         f"returning stateless function, cant find tools class for state handling found {instance_type}")
        #     if metadata:
        #         self.logger.info(f"returning metadata stateless")
        #         return (function_data, function), 0
        #     return function, 0

        self.logger.info("wrapping in higher_order_function")

        self.logger.info(f"returned fuction {specification}.{modular_id}.{function_id}")
        higher_order_function = partial(function, instance)

        if metadata:
            self.logger.info("returning metadata stateful")
            return (function_data, higher_order_function), 0

        self.logger.info("returning stateful function")
        return higher_order_function, 0

    def save_exit(self):
        self.logger.info(f"save exiting saving data to {self.config_fh.file_handler_filename} states of {self.debug=}")
        self.config_fh.add_to_save_file_handler(self.keys["debug"], str(self.debug))

    def init_mod(self, mod_name, spec='app'):
        """
        Initializes a module in a thread-safe manner by submitting the
        asynchronous initialization to the running event loop.
        """
        if '.' in mod_name:
            mod_name = mod_name.split('.')[0]
        self.run_bg_task(self.a_init_mod, mod_name, spec)
        # loop = self.loop_gard()
        # if loop:
        #     # Create a future to get the result from the coroutine
        #     future: Future = asyncio.run_coroutine_threadsafe(
        #         self.a_init_mod(mod_name, spec), loop
        #     )
        #     # Block until the result is available
        #     return future.result()
        # else:
        #     raise ValueError("Event loop is not running")
        #     # return self.loop_gard().run_until_complete(self.a_init_mod(mod_name, spec))

    def run_bg_task(self, task: Callable, *args, **kwargs) -> asyncio.Task | None:
        """
        Runs a coroutine in the background without blocking the caller.

        This is the primary method for "fire-and-forget" async tasks. It schedules
        the coroutine to run on the application's main event loop.

        Args:
            task: The coroutine function to run.
            *args: Arguments to pass to the coroutine function.
            **kwargs: Keyword arguments to pass to the coroutine function.

        Returns:
            An asyncio.Task object representing the scheduled task, or None if
            the task could not be scheduled.
        """
        if not callable(task):
            self.logger.warning("Task passed to run_bg_task is not callable!")
            return None

        if not asyncio.iscoroutinefunction(task) and not asyncio.iscoroutine(task):
            self.logger.warning(f"Task '{getattr(task, '__name__', 'unknown')}' is not a coroutine. "
                                f"Use run_bg_task_advanced for synchronous functions.")
            # Fallback to advanced runner for convenience
            self.run_bg_task_advanced(task, *args, **kwargs)
            return None

        try:
            loop = self.loop_gard()
            if not loop.is_running():
                # If the main loop isn't running, we can't create a task on it.
                # This scenario is handled by run_bg_task_advanced.
                self.logger.info("Main event loop not running. Delegating to advanced background runner.")
                return self.run_bg_task_advanced(task, *args, **kwargs)

            # Create the coroutine if it's a function
            coro = task(*args, **kwargs) if asyncio.iscoroutinefunction(task) else task

            # Create a task on the running event loop
            bg_task = loop.create_task(coro)

            # Add a callback to log exceptions from the background task
            def _log_exception(the_task: asyncio.Task):
                if not the_task.cancelled() and the_task.exception():
                    self.logger.error(f"Exception in background task '{the_task.get_name()}':",
                                      exc_info=the_task.exception())

            bg_task.add_done_callback(_log_exception)
            self.bg_tasks.append(bg_task)
            return bg_task

        except Exception as e:
            self.logger.error(f"Failed to schedule background task: {e}", exc_info=True)
            return None

    def run_bg_task_advanced(self, task: Callable, *args, **kwargs) -> threading.Thread:
        """
        Runs a task in a separate, dedicated background thread with its own event loop.

        This is ideal for:
        1. Running an async task from a synchronous context.
        2. Launching a long-running, independent operation that should not
           interfere with the main application's event loop.

        Args:
            task: The function to run (can be sync or async).
            *args: Arguments for the task.
            **kwargs: Keyword arguments for the task.

        Returns:
            The threading.Thread object managing the background execution.
        """
        if not callable(task):
            self.logger.warning("Task for run_bg_task_advanced is not callable!")
            return None

        def thread_target():
            # Each thread gets its own event loop.
            loop = asyncio.new_event_loop()
            asyncio.set_event_loop(loop)

            try:
                # Prepare the coroutine we need to run
                if asyncio.iscoroutinefunction(task):
                    coro = task(*args, **kwargs)
                elif asyncio.iscoroutine(task):
                    # It's already a coroutine object
                    coro = task
                else:
                    # It's a synchronous function, run it in an executor
                    # to avoid blocking the new event loop.
                    coro = loop.run_in_executor(None, lambda: task(*args, **kwargs))

                # Run the coroutine to completion
                result = loop.run_until_complete(coro)
                self.logger.debug(f"Advanced background task '{getattr(task, '__name__', 'unknown')}' completed.")
                if result is not None:
                    self.logger.debug(f"Task result: {str(result)[:100]}")

            except Exception as e:
                self.logger.error(f"Error in advanced background task '{getattr(task, '__name__', 'unknown')}':",
                                  exc_info=e)
            finally:
                # Cleanly shut down the event loop in this thread.
                try:
                    all_tasks = asyncio.all_tasks(loop=loop)
                    if all_tasks:
                        for t in all_tasks:
                            t.cancel()
                        loop.run_until_complete(asyncio.gather(*all_tasks, return_exceptions=True))
                finally:
                    loop.close()
                    asyncio.set_event_loop(None)

        # Create, start, and return the thread.
        # It's a daemon thread so it won't prevent the main app from exiting.
        t = threading.Thread(target=thread_target, daemon=True, name=f"BGTask-{getattr(task, '__name__', 'unknown')}")
        self.bg_tasks.append(t)
        t.start()
        return t

    # Helper method to wait for background tasks to complete (optional)
    def wait_for_bg_tasks(self, timeout=None):
        """
        Wait for all background tasks to complete.

        Args:
            timeout: Maximum time to wait (in seconds) for all tasks to complete.
                     None means wait indefinitely.

        Returns:
            bool: True if all tasks completed, False if timeout occurred
        """
        active_tasks = [t for t in self.bg_tasks if t.is_alive()]

        for task in active_tasks:
            task.join(timeout=timeout)
            if task.is_alive():
                return False

        return True

    def __call__(self, *args, **kwargs):
        return self.run(*args, **kwargs)

    def run(self, *args, request=None, running_function_coro=None, **kwargs):
        """
        Run a function with support for SSE streaming in both
        threaded and non-threaded contexts.
        """
        if running_function_coro is None:
            mn, fn = args[0]
            if self.functions.get(mn, {}).get(fn, {}).get('request_as_kwarg', False):
                kwargs["request"] = RequestData.from_dict(request)
                if 'data' in kwargs and 'data' not in self.functions.get(mn, {}).get(fn, {}).get('params', []):
                    kwargs["request"].data = kwargs["request"].body = kwargs['data']
                    del kwargs['data']
                if 'form_data' in kwargs and 'form_data' not in self.functions.get(mn, {}).get(fn, {}).get('params',
                                                                                                           []):
                    kwargs["request"].form_data = kwargs["request"].body = kwargs['form_data']
                    del kwargs['form_data']
            else:
                params = self.functions.get(mn, {}).get(fn, {}).get('params', [])
                # auto pars data and form_data to kwargs by key
                do = False
                data = {}
                if 'data' in kwargs and 'data' not in params:
                    do = True
                    data = kwargs['data']
                    del kwargs['data']
                if 'form_data' in kwargs and 'form_data' not in params:
                    do = True
                    data = kwargs['form_data']
                    del kwargs['form_data']
                if do:
                    for k in params:
                        if k in data:
                            kwargs[k] = data[k]
                            del data[k]

        # Create the coroutine
        coro = running_function_coro or self.a_run_any(*args, **kwargs)

        # Get or create an event loop
        try:
            loop = asyncio.get_event_loop()
            is_running = loop.is_running()
        except RuntimeError:
            loop = asyncio.new_event_loop()
            asyncio.set_event_loop(loop)
            is_running = False

        # If the loop is already running, run in a separate thread
        if is_running:
            # Create thread pool executor as needed
            if not hasattr(self.__class__, '_executor'):
                self.__class__._executor = ThreadPoolExecutor(max_workers=4)

            def run_in_new_thread():
                # Set up a new loop in this thread
                new_loop = asyncio.new_event_loop()
                asyncio.set_event_loop(new_loop)

                try:
                    # Run the coroutine
                    return new_loop.run_until_complete(coro)
                finally:
                    new_loop.close()

            # Run in thread and get result
            thread_result = self.__class__._executor.submit(run_in_new_thread).result()

            # Handle streaming results from thread
            if isinstance(thread_result, dict) and thread_result.get("is_stream"):
                # Create a new SSE stream in the main thread
                async def stream_from_function():
                    # Re-run the function with direct async access
                    stream_result = await self.a_run_any(*args, **kwargs)

                    if (isinstance(stream_result, Result) and
                        getattr(stream_result.result, 'data_type', None) == "stream"):
                        # Get and forward data from the original generator
                        original_gen = stream_result.result.data.get("generator")
                        if inspect.isasyncgen(original_gen):
                            async for item in original_gen:
                                yield item

                # Return a new streaming Result
                return Result.stream(
                    stream_generator=stream_from_function(),
                    headers=thread_result.get("headers", {})
                )

            result = thread_result
        else:
            # Direct execution when loop is not running
            result = loop.run_until_complete(coro)

        # Process the final result
        if isinstance(result, Result):
            if 'debug' in self.id:
                result.print()
            if getattr(result.result, 'data_type', None) == "stream":
                return result
            return result.to_api_result().model_dump(mode='json')

        return result

    def loop_gard(self):
        if self.loop is None:
            self._start_event_loop()
            self.loop = asyncio.get_event_loop()
        if self.loop.is_closed():
            self.loop = asyncio.get_event_loop()
        return self.loop

    async def a_init_mod(self, mod_name, spec='app'):
        mod = self.save_load(mod_name, spec=spec)
        if hasattr(mod, "__initobj") and not mod.async_initialized:
            await mod
        return mod


    def load_mod(self, mod_name: str, mlm='I', **kwargs):

        action_list_helper = ['I (inplace load dill on error python)',
                              # 'C (coppy py file to runtime dir)',
                              # 'S (save py file to dill)',
                              # 'CS (coppy and save py file)',
                              # 'D (development mode, inplace load py file)'
                              ]
        action_list = {"I": lambda: self.inplace_load_instance(mod_name, **kwargs),
                       "C": lambda: self._copy_load(mod_name, **kwargs)
                       }

        try:
            if mlm in action_list:

                return action_list.get(mlm)()
            else:
                self.logger.critical(
                    f"config mlm must be {' or '.join(action_list_helper)} is {mlm=}")
                raise ValueError(f"config mlm must be {' or '.join(action_list_helper)} is {mlm=}")
        except ValueError as e:
            self.logger.warning(Style.YELLOW(f"Error Loading Module '{mod_name}', with error :{e}"))
            self.debug_rains(e)
        except ImportError as e:
            self.logger.error(Style.YELLOW(f"Error Loading Module '{mod_name}', with error :{e}"))
            self.debug_rains(e)
        except Exception as e:
            self.logger.critical(Style.RED(f"Error Loading Module '{mod_name}', with critical error :{e}"))
            print(Style.RED(f"Error Loading Module '{mod_name}'"))
            self.debug_rains(e)

        return Result.default_internal_error(info="info's in logs.")

    async def load_external_mods(self):
        for mod_path in os.getenv("EXTERNAL_PATH_RUNNABLE", '').split(','):
            if mod_path:
                await self.load_all_mods_in_file(mod_path)

    async def load_all_mods_in_file(self, working_dir="mods"):
        print(f"LOADING ALL MODS FROM FOLDER : {working_dir}")
        t0 = time.perf_counter()
        # Get the list of all modules
        module_list = self.get_all_mods(working_dir)
        open_modules = self.functions.keys()
        start_len = len(open_modules)

        for om in open_modules:
            if om in module_list:
                module_list.remove(om)

        tasks: set[Task] = set()

        _ = {tasks.add(asyncio.create_task(asyncio.to_thread(self.save_load, mod, 'app'))) for mod in module_list}
        for t in asyncio.as_completed(tasks):
            try:
                result = await t
                if hasattr(result, 'Name'):
                    self.print('Opened :', result.Name)
                elif hasattr(result, 'name'):
                    if hasattr(result, 'async_initialized'):
                        if not result.async_initialized:
                            async def _():
                                try:
                                    if asyncio.iscoroutine(result):
                                        await result
                                    if hasattr(result, 'Name'):
                                        self.print('Opened :', result.Name)
                                    elif hasattr(result, 'name'):
                                        self.print('Opened :', result.name)
                                except Exception as e:
                                    self.debug_rains(e)
                                    if hasattr(result, 'Name'):
                                        self.print('Error opening :', result.Name)
                                    elif hasattr(result, 'name'):
                                        self.print('Error opening :', result.name)
                            asyncio.create_task(_())
                        else:
                            self.print('Opened :', result.name)
                else:
                    if result:
                        self.print('Opened :', result)
            except Exception as e:
                self.logger.error(Style.RED(f"An Error occurred while opening all modules error: {str(e)}"))
                self.debug_rains(e)
        opened = len(self.functions.keys()) - start_len

        self.logger.info(f"Opened {opened} modules in {time.perf_counter() - t0:.2f}s")
        return f"Opened {opened} modules in {time.perf_counter() - t0:.2f}s"

    def get_all_mods(self, working_dir="mods", path_to="./runtime", use_wd=True):
        self.logger.info(f"collating all mods in working directory {working_dir}")

        pr = "_dev" if self.dev_modi else ""
        if working_dir == "mods" and use_wd:
            working_dir = f"{self.start_dir}/mods{pr}"
        elif use_wd:
            pass
        else:
            w_dir = self.id.replace(".", "_")
            working_dir = f"{path_to}/{w_dir}/mod_lib{pr}/"
        res = os.listdir(working_dir)

        self.logger.info(f"found : {len(res)} files")

        def do_helper(_mod):
            if "mainTool" in _mod:
                return False
            # if not _mod.endswith(".py"):
            #     return False
            if _mod.startswith("__"):
                return False
            if _mod.startswith("."):
                return False
            return not _mod.startswith("test_")

        def r_endings(word: str):
            if word.endswith(".py"):
                return word[:-3]
            return word

        mods_list = list(map(r_endings, filter(do_helper, res)))

        self.logger.info(f"found : {len(mods_list)} Modules")
        return mods_list

    def remove_all_modules(self, delete=False):
        for mod in list(self.functions.keys()):
            self.logger.info(f"closing: {mod}")
            self.remove_mod(mod, delete=delete)

    def remove_mod(self, mod_name, spec='app', delete=True):
        if mod_name not in self.functions:
            self.logger.info(f"mod not active {mod_name}")
            return

        on_exit = self.functions[mod_name].get("on_exit")
        self.logger.info(f"closing: {on_exit}")
        def helper():
            if f"{spec}_instance" in self.functions[mod_name]:
                del self.functions[mod_name][f"{spec}_instance"]
            if f"{spec}_instance_type" in self.functions[mod_name]:
                del self.functions[mod_name][f"{spec}_instance_type"]

        if on_exit is None and self.functions[mod_name].get(f"{spec}_instance_type", "").endswith("/BC"):
            instance = self.functions[mod_name].get(f"{spec}_instance", None)
            if instance is not None and hasattr(instance, 'on_exit'):
                if asyncio.iscoroutinefunction(instance.on_exit):
                    self.exit_tasks.append(instance.on_exit)
                else:
                    instance.on_exit()

        if on_exit is None and delete:
            self.functions[mod_name] = {}
            del self.functions[mod_name]
            return
        if on_exit is None:
            helper()
            return

        i = 1

        for j, f in enumerate(on_exit):
            try:
                f_, e = self.get_function((mod_name, f), state=True, specification=spec, i=j)
                if e == 0:
                    self.logger.info(Style.GREY(f"Running On exit {f} {i}/{len(on_exit)}"))
                    if asyncio.iscoroutinefunction(f_):
                        self.exit_tasks.append(f_)
                        o = None
                    else:
                        o = f_()
                    if o is not None:
                        self.print(f"Function On Exit result: {o}")
                else:
                    self.logger.warning("closing function not found")
            except Exception as e:
                self.logger.debug(
                    Style.YELLOW(Style.Bold(f"modular:{mod_name}.{f} on_exit error {i}/{len(on_exit)} -> {e}")))

                self.debug_rains(e)
            finally:
                i += 1

        helper()

        if delete:
            self.functions[mod_name] = {}
            del self.functions[mod_name]

    async def a_remove_all_modules(self, delete=False):
        for mod in list(self.functions.keys()):
            self.logger.info(f"closing: {mod}")
            await self.a_remove_mod(mod, delete=delete)

    async def a_remove_mod(self, mod_name, spec='app', delete=True):
        if mod_name not in self.functions:
            self.logger.info(f"mod not active {mod_name}")
            return
        on_exit = self.functions[mod_name].get("on_exit")
        self.logger.info(f"closing: {on_exit}")
        def helper():
            if f"{spec}_instance" in self.functions[mod_name]:
                del self.functions[mod_name][f"{spec}_instance"]
            if f"{spec}_instance_type" in self.functions[mod_name]:
                del self.functions[mod_name][f"{spec}_instance_type"]

        if on_exit is None and self.functions[mod_name].get(f"{spec}_instance_type", "").endswith("/BC"):
            instance = self.functions[mod_name].get(f"{spec}_instance", None)
            if instance is not None and hasattr(instance, 'on_exit'):
                if asyncio.iscoroutinefunction(instance.on_exit):
                    await instance.on_exit()
                else:
                    instance.on_exit()

        if on_exit is None and delete:
            self.functions[mod_name] = {}
            del self.functions[mod_name]
            return
        if on_exit is None:
            helper()
            return

        i = 1
        for f in on_exit:
            try:
                e = 1
                if isinstance(f, str):
                    f_, e = self.get_function((mod_name, f), state=True, specification=spec)
                elif isinstance(f, Callable):
                    f_, e, f  = f, 0, f.__name__
                if e == 0:
                    self.logger.info(Style.GREY(f"Running On exit {f} {i}/{len(on_exit)}"))
                    if asyncio.iscoroutinefunction(f_):
                        o = await f_()
                    else:
                        o = f_()
                    if o is not None:
                        self.print(f"Function On Exit result: {o}")
                else:
                    self.logger.warning("closing function not found")
            except Exception as e:
                self.logger.debug(
                    Style.YELLOW(Style.Bold(f"modular:{mod_name}.{f} on_exit error {i}/{len(on_exit)} -> {e}")))
                self.debug_rains(e)
            finally:
                i += 1

        helper()

        if delete:
            self.functions[mod_name] = {}
            del self.functions[mod_name]

    def exit(self, remove_all=True):
        if not self.alive:
            return
        if self.args_sto.debug:
            self.hide_console()
        self.disconnect()
        if remove_all:
            self.remove_all_modules()
        self.logger.info("Exiting ToolBox interface")
        self.alive = False
        self.called_exit = True, time.time()
        self.save_exit()
        if hasattr(self, 'root_blob_storage') and self.root_blob_storage:
            self.root_blob_storage.exit()
        try:
            self.config_fh.save_file_handler()
        except SystemExit:
            print("If u ar testing this is fine else ...")

        if hasattr(self, 'daemon_app'):
            import threading

            for thread in threading.enumerate()[::-1]:
                if thread.name == "MainThread":
                    continue
                try:
                    with Spinner(f"closing Thread {thread.name:^50}|", symbols="s", count_down=True,
                                 time_in_s=0.751 if not self.debug else 0.6):
                        thread.join(timeout=0.751 if not self.debug else 0.6)
                except TimeoutError as e:
                    self.logger.error(f"Timeout error on exit {thread.name} {str(e)}")
                    print(str(e), f"Timeout {thread.name}")
                except KeyboardInterrupt:
                    print("Unsave Exit")
                    break
        if hasattr(self, 'loop') and self.loop is not None:
            with Spinner("closing Event loop:", symbols="+"):
                self.loop.stop()

    async def a_exit(self):

        import inspect
        self.sprint(f"exit requested from: {inspect.stack()[1].filename}::{inspect.stack()[1].lineno} function: {inspect.stack()[1].function}")

        # Cleanup session before removing modules
        try:
            if hasattr(self, 'session') and self.session is not None:
                await self.session.cleanup()
        except Exception as e:
            self.logger.debug(f"Session cleanup error (ignored): {e}")

        await self.a_remove_all_modules(delete=True)
        results = await asyncio.gather(
            *[asyncio.create_task(f()) for f in self.exit_tasks if asyncio.iscoroutinefunction(f)])
        for result in results:
            self.print(f"Function On Exit result: {result}")
        self.exit(remove_all=False)

    def save_load(self, modname, spec='app'):
        self.logger.debug(f"Save load module {modname}")
        if not modname:
            self.logger.warning("no filename specified")
            return False
        try:
            return self.load_mod(modname, spec=spec)
        except ModuleNotFoundError as e:
            self.logger.error(Style.RED(f"Module {modname} not found"))
            self.debug_rains(e)

        return False

    def get_function(self, name: Enum or tuple, **kwargs):
        """
        Kwargs for _get_function
            metadata:: return the registered function dictionary
                stateless: (function_data, None), 0
                stateful: (function_data, higher_order_function), 0
            state::boolean
                specification::str default app
        """
        if isinstance(name, tuple):
            return self._get_function(None, as_str=name, **kwargs)
        else:
            return self._get_function(name, **kwargs)

    async def a_run_function(self, mod_function_name: Enum or tuple,
                             tb_run_function_with_state=True,
                             tb_run_with_specification='app',
                             args_=None,
                             kwargs_=None,
                             *args,
                             **kwargs) -> Result:

        if kwargs_ is not None and not kwargs:
            kwargs = kwargs_
        if args_ is not None and not args:
            args = args_
        if isinstance(mod_function_name, tuple):
            modular_name, function_name = mod_function_name
        elif isinstance(mod_function_name, list):
            modular_name, function_name = mod_function_name[0], mod_function_name[1]
        elif isinstance(mod_function_name, Enum):
            modular_name, function_name = mod_function_name.__class__.NAME.value, mod_function_name.value
        else:
            raise TypeError("Unknown function type")

        if tb_run_with_specification == 'ws_internal':
            modular_name = modular_name.split('/')[0]
            if not self.mod_online(modular_name, installed=True):
                self.get_mod(modular_name)
            handler_id, event_name = mod_function_name
            if handler_id in self.websocket_handlers and event_name in self.websocket_handlers[handler_id]:
                handler_func = self.websocket_handlers[handler_id][event_name]
                try:
                    # Führe den asynchronen Handler aus
                    if inspect.iscoroutinefunction(handler_func):
                        await handler_func(self, **kwargs)
                    else:
                        handler_func(self, **kwargs)  # Für synchrone Handler
                    return Result.ok(info=f"WS handler '{event_name}' executed.")
                except Exception as e:
                    self.logger.error(f"Error in WebSocket handler '{handler_id}/{event_name}': {e}", exc_info=True)
                    return Result.default_internal_error(info=str(e))
            else:
                # Kein Handler registriert, aber das ist kein Fehler (z.B. on_connect ist optional)
                return Result.ok(info=f"No WS handler for '{event_name}'.")

        if not self.mod_online(modular_name, installed=True):
            self.get_mod(modular_name)

        function_data, error_code = self.get_function(mod_function_name, state=tb_run_function_with_state,
                                                      metadata=True, specification=tb_run_with_specification)
        self.logger.info(f"Received fuction : {mod_function_name}, with execode: {error_code}")
        if error_code == 404:
            mod = self.get_mod(modular_name)
            if hasattr(mod, "async_initialized") and not mod.async_initialized:
                await mod
            function_data, error_code = self.get_function(mod_function_name, state=tb_run_function_with_state,
                                                          metadata=True, specification=tb_run_with_specification)

        if error_code == 404:
            self.logger.warning(Style.RED("Function Not Found"))
            return (Result.default_user_error(interface=self.interface_type,
                                              exec_code=404,
                                              info="function not found function is not decorated").
                    set_origin(mod_function_name))

        if error_code == 300:
            return Result.default_internal_error(interface=self.interface_type,
                                                 info=f"module {modular_name}"
                                                      f" has no state (instance)").set_origin(mod_function_name)

        if error_code != 0:
            return Result.default_internal_error(interface=self.interface_type,
                                                 exec_code=error_code,
                                                 info=f"Internal error"
                                                      f" {modular_name}."
                                                      f"{function_name}").set_origin(mod_function_name)

        if not tb_run_function_with_state:
            function_data, _ = function_data
            function = function_data.get('func')
        else:
            function_data, function = function_data

        if not function:
            self.logger.warning(Style.RED(f"Function {function_name} not found"))
            return Result.default_internal_error(interface=self.interface_type,
                                                 exec_code=404,
                                                 info="function not found function").set_origin(mod_function_name)

        self.logger.info("Profiling function")
        t0 = time.perf_counter()
        if asyncio.iscoroutinefunction(function):
            return await self.a_fuction_runner(function, function_data, args, kwargs, t0)
        else:
            return self.fuction_runner(function, function_data, args, kwargs, t0)


    def run_function(self, mod_function_name: Enum or tuple,
                     tb_run_function_with_state=True,
                     tb_run_with_specification='app',
                     args_=None,
                     kwargs_=None,
                     *args,
                     **kwargs) -> Result:

        if kwargs_ is not None and not kwargs:
            kwargs = kwargs_
        if args_ is not None and not args:
            args = args_
        if isinstance(mod_function_name, tuple):
            modular_name, function_name = mod_function_name
        elif isinstance(mod_function_name, list):
            modular_name, function_name = mod_function_name[0], mod_function_name[1]
        elif isinstance(mod_function_name, Enum):
            modular_name, function_name = mod_function_name.__class__.NAME.value, mod_function_name.value
        else:
            raise TypeError("Unknown function type")

        if not self.mod_online(modular_name, installed=True):
            self.get_mod(modular_name)

        if tb_run_with_specification == 'ws_internal':
            handler_id, event_name = mod_function_name
            if handler_id in self.websocket_handlers and event_name in self.websocket_handlers[handler_id]:
                handler_func = self.websocket_handlers[handler_id][event_name]
                try:
                    # Führe den asynchronen Handler aus
                    if inspect.iscoroutinefunction(handler_func):
                        return self.loop.run_until_complete(handler_func(self, **kwargs))
                    else:
                        handler_func(self, **kwargs)  # Für synchrone Handler
                    return Result.ok(info=f"WS handler '{event_name}' executed.")
                except Exception as e:
                    self.logger.error(f"Error in WebSocket handler '{handler_id}/{event_name}': {e}", exc_info=True)
                    return Result.default_internal_error(info=str(e))
            else:
                # Kein Handler registriert, aber das ist kein Fehler (z.B. on_connect ist optional)
                return Result.ok(info=f"No WS handler for '{event_name}'.")

        function_data, error_code = self.get_function(mod_function_name, state=tb_run_function_with_state,
                                                      metadata=True, specification=tb_run_with_specification)
        self.logger.info(f"Received fuction : {mod_function_name}, with execode: {error_code}")
        if error_code == 1 or error_code == 3 or error_code == 400:
            self.get_mod(modular_name)
            function_data, error_code = self.get_function(mod_function_name, state=tb_run_function_with_state,
                                                          metadata=True, specification=tb_run_with_specification)

        if error_code == 2:
            self.logger.warning(Style.RED("Function Not Found"))
            return (Result.default_user_error(interface=self.interface_type,
                                              exec_code=404,
                                              info="function not found function is not decorated").
                    set_origin(mod_function_name))

        if error_code == -1:
            return Result.default_internal_error(interface=self.interface_type,
                                                 info=f"module {modular_name}"
                                                      f" has no state (instance)").set_origin(mod_function_name)

        if error_code != 0:
            return Result.default_internal_error(interface=self.interface_type,
                                                 exec_code=error_code,
                                                 info=f"Internal error"
                                                      f" {modular_name}."
                                                      f"{function_name}").set_origin(mod_function_name)

        if not tb_run_function_with_state:
            function_data, _ = function_data
            function = function_data.get('func')
        else:
            function_data, function = function_data

        if not function:
            self.logger.warning(Style.RED(f"Function {function_name} not found"))
            return Result.default_internal_error(interface=self.interface_type,
                                                 exec_code=404,
                                                 info="function not found function").set_origin(mod_function_name)

        self.logger.info("Profiling function")
        t0 = time.perf_counter()
        if asyncio.iscoroutinefunction(function):
            try:
                return asyncio.run(self.a_fuction_runner(function, function_data, args, kwargs, t0))
            except RuntimeError:
                try:
                    return self.loop.run_until_complete(self.a_fuction_runner(function, function_data, args, kwargs, t0))
                except RuntimeError:
                    pass
            raise ValueError(f"Fuction {function_name} is Async use a_run_any")
        else:
            return self.fuction_runner(function, function_data, args, kwargs, t0)

    def run_a_from_sync(self, function, *args, **kwargs):
        # Initialize self.loop if not already set.
        if self.loop is None:
            try:
                self.loop = asyncio.get_running_loop()
            except RuntimeError:
                self.loop = asyncio.new_event_loop()

        # If the loop is running, offload the coroutine to a new thread.
        if self.loop.is_running():
            result_future = Future()

            def run_in_new_loop():
                new_loop = asyncio.new_event_loop()
                asyncio.set_event_loop(new_loop)
                try:
                    result = new_loop.run_until_complete(function(*args, **kwargs))
                    result_future.set_result(result)
                except Exception as e:
                    result_future.set_exception(e)
                finally:
                    new_loop.close()

            thread = threading.Thread(target=run_in_new_loop)
            thread.start()
            thread.join()  # Block until the thread completes.
            return result_future.result()
        else:
            # If the loop is not running, schedule and run the coroutine directly.
            future = self.loop.create_task(function(*args, **kwargs))
            return self.loop.run_until_complete(future)

    def fuction_runner(self, function, function_data: dict, args: list, kwargs: dict, t0=.0):

        parameters = function_data.get('params')
        modular_name = function_data.get('module_name')
        function_name = function_data.get('func_name')
        row = function_data.get('row')
        mod_function_name = f"{modular_name}.{function_name}"

        if_self_state = 1 if 'self' in parameters else 0

        try:
            if len(parameters) == 0:
                res = function()
            elif len(parameters) == len(args) + if_self_state:
                res = function(*args)
            elif len(parameters) == len(kwargs.keys()) + if_self_state:
                res = function(**kwargs)
            else:
                res = function(*args, **kwargs)
            self.logger.info(f"Execution done in {time.perf_counter()-t0:.4f}")
            if isinstance(res, Result):
                formatted_result = res
                if formatted_result.origin is None:
                    formatted_result.set_origin(mod_function_name)
            elif isinstance(res, ApiResult):
                formatted_result = res
                if formatted_result.origin is None:
                    formatted_result.as_result().set_origin(mod_function_name).to_api_result()
            elif row:
                formatted_result = res
            else:
                # Wrap the result in a Result object
                formatted_result = Result.ok(
                    interface=self.interface_type,
                    data_info="Auto generated result",
                    data=res,
                    info="Function executed successfully"
                ).set_origin(mod_function_name)
            if not row:
                self.logger.info(
                    f"Function Exec code: {formatted_result.info.exec_code} Info's: {formatted_result.info.help_text}")
            else:
                self.logger.info(
                    f"Function Exec data: {formatted_result}")
        except Exception as e:
            self.logger.error(
                Style.YELLOW(Style.Bold(
                    f"! Function ERROR: in {modular_name}.{function_name}")))
            # Wrap the exception in a Result object
            formatted_result = Result.default_internal_error(info=str(e)).set_origin(mod_function_name)
            # res = formatted_result
            self.logger.error(
                f"Function {modular_name}.{function_name}"
                f" executed wit an error {str(e)}, {type(e)}")
            self.debug_rains(e)
            self.print(f"! Function ERROR: in {modular_name}.{function_name} ")



        else:
            self.print_ok()

            self.logger.info(
                f"Function {modular_name}.{function_name}"
                f" executed successfully")

        return formatted_result

    async def a_fuction_runner(self, function, function_data: dict, args: list, kwargs: dict, t0=.0):

        parameters = function_data.get('params')
        modular_name = function_data.get('module_name')
        function_name = function_data.get('func_name')
        row = function_data.get('row')
        mod_function_name = f"{modular_name}.{function_name}"

        if_self_state = 1 if 'self' in parameters else 0

        try:
            if len(parameters) == 0:
                res = await function()
            elif len(parameters) == len(args) + if_self_state:
                res = await function(*args)
            elif len(parameters) == len(kwargs.keys()) + if_self_state:
                res = await function(**kwargs)
            else:
                res = await function(*args, **kwargs)
            self.logger.info(f"Execution done in {time.perf_counter()-t0:.4f}")
            if isinstance(res, Result):
                formatted_result = res
                if formatted_result.origin is None:
                    formatted_result.set_origin(mod_function_name)
            elif isinstance(res, ApiResult):
                formatted_result = res
                if formatted_result.origin is None:
                    formatted_result.as_result().set_origin(mod_function_name).to_api_result()
            elif row:
                formatted_result = res
            else:
                # Wrap the result in a Result object
                formatted_result = Result.ok(
                    interface=self.interface_type,
                    data_info="Auto generated result",
                    data=res,
                    info="Function executed successfully"
                ).set_origin(mod_function_name)
            if not row:
                self.logger.info(
                    f"Function Exec code: {formatted_result.info.exec_code} Info's: {formatted_result.info.help_text}")
            else:
                self.logger.info(
                    f"Function Exec data: {formatted_result}")
        except Exception as e:
            self.logger.error(
                Style.YELLOW(Style.Bold(
                    f"! Function ERROR: in {modular_name}.{function_name}")))
            # Wrap the exception in a Result object
            formatted_result = Result.default_internal_error(info=str(e)).set_origin(mod_function_name)
            # res = formatted_result
            self.logger.error(
                f"Function {modular_name}.{function_name}"
                f" executed wit an error {str(e)}, {type(e)}")
            self.debug_rains(e)

        else:
            self.print_ok()

            self.logger.info(
                f"Function {modular_name}.{function_name}"
                f" executed successfully")

        return formatted_result

    async def run_http(self, mod_function_name: Enum or str or tuple, function_name=None,
                       args_=None,
                       kwargs_=None, method="GET",
                       *args, **kwargs):
        if kwargs_ is not None and not kwargs:
            kwargs = kwargs_
        if args_ is not None and not args:
            args = args_

        modular_name = mod_function_name
        function_name = function_name

        if isinstance(mod_function_name, str) and isinstance(function_name, str):
            mod_function_name = (mod_function_name, function_name)

        if isinstance(mod_function_name, tuple):
            modular_name, function_name = mod_function_name
        elif isinstance(mod_function_name, list):
            modular_name, function_name = mod_function_name[0], mod_function_name[1]
        elif isinstance(mod_function_name, Enum):
            modular_name, function_name = mod_function_name.__class__.NAME.value, mod_function_name.value

        self.logger.info(f"getting function : {modular_name}.{function_name} from http {self.session.base}")
        r = await self.session.fetch(f"/api/{modular_name}/{function_name}{'?' + args_ if args_ is not None else ''}",
                                     data=kwargs, method=method)
        try:
            if not r:
                print("§ Session server Offline!", self.session.base)
                return Result.default_internal_error(info="Session fetch failed").as_dict()

            content_type = r.headers.get('Content-Type', '').lower()

            if 'application/json' in content_type:
                try:
                    return r.json()
                except Exception as e:
                    print(f"⚠ JSON decode error: {e}")
                    # Fallback to text if JSON decoding fails
                    text = r.text
            else:
                text = r.text

            if isinstance(text, Callable):
                if asyncio.iscoroutinefunction(text):
                    text = await text()
                else:
                    text = text()

            # Attempt YAML
            if 'yaml' in content_type or text.strip().startswith('---'):
                try:
                    import yaml
                    return yaml.safe_load(text)
                except Exception as e:
                    print(f"⚠ YAML decode error: {e}")

            # Attempt XML
            if 'xml' in content_type or text.strip().startswith('<?xml'):
                try:
                    import xmltodict
                    return xmltodict.parse(text)
                except Exception as e:
                    print(f"⚠ XML decode error: {e}")

            # Fallback: return plain text
            return Result.default_internal_error(data={'raw_text': text, 'content_type': content_type}).as_dict()

        except Exception as e:
            print("❌ Fatal error during API call:", e)
            self.debug_rains(e)
            return Result.default_internal_error(str(e)).as_dict()

    def run_local(self, *args, **kwargs):
        return self.run_any(*args, **kwargs)

    async def a_run_local(self, *args, **kwargs):
        return await self.a_run_any(*args, **kwargs)

    def run_any(self, mod_function_name: Enum or str or tuple, backwords_compability_variabel_string_holder=None,
                get_results=False, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None,
                kwargs_=None,
                *args, **kwargs):

        # if self.debug:
        #     self.logger.info(f'Called from: {getouterframes(currentframe(), 2)}')

        if kwargs_ is not None and not kwargs:
            kwargs = kwargs_
        if args_ is not None and not args:
            args = args_

        if isinstance(mod_function_name, str) and backwords_compability_variabel_string_holder is None:
            backwords_compability_variabel_string_holder = mod_function_name.split('.')[-1]
            mod_function_name = mod_function_name.replace(f".{backwords_compability_variabel_string_holder}", "")

        if isinstance(mod_function_name, str) and isinstance(backwords_compability_variabel_string_holder, str):
            mod_function_name = (mod_function_name, backwords_compability_variabel_string_holder)

        res: Result = self.run_function(mod_function_name,
                                        tb_run_function_with_state=tb_run_function_with_state,
                                        tb_run_with_specification=tb_run_with_specification,
                                        args_=args, kwargs_=kwargs).as_result()
        if isinstance(res, ApiResult):
            res = res.as_result()

        if isinstance(res, Result) and res.bg_task is not None:
            self.run_bg_task(res.bg_task)

        if self.debug:
            res.log(show_data=False)

        if not get_results and isinstance(res, Result):
            return res.get()

        if get_results and not isinstance(res, Result):
            return Result.ok(data=res)

        return res

    async def a_run_any(self, mod_function_name: Enum or str or tuple,
                        backwords_compability_variabel_string_holder=None,
                        get_results=False, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None,
                        kwargs_=None,
                        *args, **kwargs):

        # if self.debug:
        #     self.logger.info(f'Called from: {getouterframes(currentframe(), 2)}')

        if kwargs_ is not None and not kwargs:
            kwargs = kwargs_
        if args_ is not None and not args:
            args = args_

        if isinstance(mod_function_name, str) and backwords_compability_variabel_string_holder is None:
            backwords_compability_variabel_string_holder = mod_function_name.split('.')[-1]
            mod_function_name = mod_function_name.replace(f".{backwords_compability_variabel_string_holder}", "")

        if isinstance(mod_function_name, str) and isinstance(backwords_compability_variabel_string_holder, str):
            mod_function_name = (mod_function_name, backwords_compability_variabel_string_holder)

        res: Result = await self.a_run_function(mod_function_name,
                                                tb_run_function_with_state=tb_run_function_with_state,
                                                tb_run_with_specification=tb_run_with_specification,
                                                args_=args, kwargs_=kwargs)
        if isinstance(res, ApiResult):
            res = res.as_result()

        if isinstance(res, Result) and res.bg_task is not None:
            self.run_bg_task(res.bg_task)

        if self.debug:
            res.print()
            res.log(show_data=False) if isinstance(res, Result) else self.logger.debug(res)
        if not get_results and isinstance(res, Result):
            return res.get()

        if get_results and not isinstance(res, Result):
            return Result.ok(data=res)

        return res


    def web_context(self):
        if self._web_context is None:
            try:
                self._web_context = open("./dist/helper.html", encoding="utf-8").read()
            except Exception as e:
                self.logger.error(f"Could not load web context: {e}")
                self._web_context = "<div><h1>Web Context not found</h1></div>"
        return self._web_context

    def get_mod(self, name, spec='app') -> ModuleType or MainToolType:
        if spec != "app":
            self.print(f"Getting Module {name} spec: {spec}")
        if name not in self.functions:
            mod = self.save_load(name, spec=spec)
            if mod is False or (isinstance(mod, Result) and mod.is_error()):
                self.logger.warning(f"Could not find {name} in {list(self.functions.keys())}")
                raise ValueError(f"Could not find {name} in {list(self.functions.keys())} pleas install the module, or its posibly broken use --debug for infos")
        # private = self.functions[name].get(f"{spec}_private")
        # if private is not None:
        #     if private and spec != 'app':
        #         raise ValueError("Module is private")
        if name not in self.functions:
            self.logger.warning(f"Module '{name}' is not found")
            return None
        instance = self.functions[name].get(f"{spec}_instance")
        if instance is None:
            return self.load_mod(name, spec=spec)
        return self.functions[name].get(f"{spec}_instance")

    def print(self, text="", *args, **kwargs):
        # self.logger.info(f"Output : {text}")
        if 'live' in self.id:
            return

        flush = kwargs.pop('flush', True)
        if self.sprint(None):
            print(Style.CYAN(f"System${self.id}:"), end=" ", flush=flush)
        print(text, *args, **kwargs, flush=flush)

    def sprint(self, text="", show_system=True, *args, **kwargs):
        if text is None:
            return True
        if 'live' in self.id:
            return
        flush = kwargs.pop('flush', True)
        # self.logger.info(f"Output : {text}")
        if show_system:
            print(Style.CYAN(f"System${self.id}:"), end=" ", flush=flush)
        if isinstance(text, str) and kwargs == {} and text:
            stram_print(text + ' '.join(args))
            print()
        else:
            print(text, *args, **kwargs, flush=flush)

    # ----------------------------------------------------------------
    # Decorators for the toolbox

    def reload_mod(self, mod_name, spec='app', is_file=True, loc="toolboxv2.mods."):
        self.remove_mod(mod_name, delete=True)
        if mod_name not in self.modules:
            self.logger.warning(f"Module '{mod_name}' is not found")
            return
        if hasattr(self.modules[mod_name], 'reload_save') and self.modules[mod_name].reload_save:
            def reexecute_module_code(x):
                return x
        else:
            def reexecute_module_code(module_name):
                if isinstance(module_name, str):
                    module = import_module(module_name)
                else:
                    module = module_name
                # Get the source code of the module
                try:
                    source = inspect.getsource(module)
                except Exception:
                    # print(f"No source for {str(module_name).split('from')[0]}: {e}")
                    return module
                # Compile the source code
                try:
                    code = compile(source, module.__file__, 'exec')
                    # Execute the code in the module's namespace
                    exec(code, module.__dict__)
                except Exception:
                    # print(f"No source for {str(module_name).split('from')[0]}: {e}")
                    pass
                return module

        if not is_file:
            mods = self.get_all_mods("./mods/" + mod_name)
            def recursive_reload(package_name):
                package = import_module(package_name)

                # First, reload all submodules
                if hasattr(package, '__path__'):
                    for _finder, name, _ispkg in pkgutil.walk_packages(package.__path__, package.__name__ + "."):
                        try:
                            mod = import_module(name)
                            reexecute_module_code(mod)
                            reload(mod)
                        except Exception as e:
                            print(f"Error reloading module {name}: {e}")
                            break

                # Finally, reload the package itself
                reexecute_module_code(package)
                reload(package)

            for mod in mods:
                if mod.endswith(".txt") or mod.endswith(".yaml"):
                    continue
                try:
                    recursive_reload(loc + mod_name + '.' + mod)
                    self.print(f"Reloaded {mod_name}.{mod}")
                except ImportError:
                    self.print(f"Could not load {mod_name}.{mod}")
        reexecute_module_code(self.modules[mod_name])
        if mod_name in self.functions:
            if "on_exit" in self.functions[mod_name]:
                self.functions[mod_name]["on_exit"] = []
            if "on_start" in self.functions[mod_name]:
                self.functions[mod_name]["on_start"] = []
        self.inplace_load_instance(mod_name, spec=spec, mfo=reload(self.modules[mod_name]) if mod_name in self.modules else None)

    def watch_mod(self, mod_name, spec='app', loc="toolboxv2.mods.", use_thread=True, path_name=None, on_reload=None):
        if path_name is None:
            path_name = mod_name
        is_file = os.path.isfile(self.start_dir + '/mods/' + path_name + '.py')
        import watchfiles
        def helper():
            paths = f'mods/{path_name}' + ('.py' if is_file else '')
            self.logger.info(f'Watching Path: {paths}')
            try:
                for changes in watchfiles.watch(paths):
                    if not changes:
                        continue
                    self.reload_mod(mod_name, spec, is_file, loc)
                    if on_reload:
                        on_reload()
            except FileNotFoundError:
                self.logger.warning(f"Path {paths} not found")

        if not use_thread:
            helper()
        else:
            threading.Thread(target=helper, daemon=True).start()

    def _register_function(self, module_name, func_name, data):
        if module_name not in self.functions:
            self.functions[module_name] = {}
        if func_name in self.functions[module_name]:
            self.print(f"Overriding function {func_name} from {module_name}", end="\r")
            self.functions[module_name][func_name] = data
        else:
            self.functions[module_name][func_name] = data

    def _create_decorator(self, type_: str,
                          name: str = "",
                          mod_name: str = "",
                          level: int = -1,
                          restrict_in_virtual_mode: bool = False,
                          api: bool = False,
                          helper: str = "",
                          version: str or None = None,
                          initial: bool=False,
                          exit_f: bool=False,
                          test: bool=True,
                          samples:list[dict[str, Any]] | None=None,
                          state:bool | None=None,
                          pre_compute:Callable | None=None,
                          post_compute:Callable[[], Result] | None=None,
                          api_methods:list[str] | None=None,
                          memory_cache: bool=False,
                          file_cache: bool=False,
                          request_as_kwarg: bool=False,
                          row: bool=False,
                          memory_cache_max_size:int=100,
                          memory_cache_ttl:int=300,
                          websocket_handler: str | None = None,
                          ):

        if isinstance(type_, Enum):
            type_ = type_.value

        if memory_cache and file_cache:
            raise ValueError("Don't use both cash at the same time for the same fuction")

        use_cache = memory_cache or file_cache
        cache = {}
        if file_cache:
            cache = FileCache(folder=self.data_dir + f'\\cache\\{mod_name}\\',
                              filename=self.data_dir + f'\\cache\\{mod_name}\\{name}cache.db')
        if memory_cache:
            cache = MemoryCache(maxsize=memory_cache_max_size, ttl=memory_cache_ttl)

        version = self.version if version is None else self.version + ':' + version

        def a_additional_process(func):

            async def executor(*args, **kwargs):

                if pre_compute is not None:
                    args, kwargs = await pre_compute(*args, **kwargs)
                if asyncio.iscoroutinefunction(func):
                    result = await func(*args, **kwargs)
                else:
                    result = func(*args, **kwargs)
                if post_compute is not None:
                    result = await post_compute(result)
                if row:
                    return result
                if not isinstance(result, Result):
                    result = Result.ok(data=result)
                if result.origin is None:
                    result.set_origin((mod_name if mod_name else func.__module__.split('.')[-1]
                                       , name if name else func.__name__
                                       , type_))
                if result.result.data_to == ToolBoxInterfaces.native.name:
                    result.result.data_to = ToolBoxInterfaces.remote if api else ToolBoxInterfaces.native
                # Wenden Sie die to_api_result Methode auf das Ergebnis an, falls verfügbar
                if api and hasattr(result, 'to_api_result'):
                    return result.to_api_result()
                return result

            @wraps(func)
            async def wrapper(*args, **kwargs):

                if not use_cache:
                    return await executor(*args, **kwargs)

                try:
                    cache_key = (f"{mod_name if mod_name else func.__module__.split('.')[-1]}"
                                 f"-{func.__name__}-{str(args)},{str(kwargs.items())}")
                except ValueError:
                    cache_key = (f"{mod_name if mod_name else func.__module__.split('.')[-1]}"
                                 f"-{func.__name__}-{bytes(args)},{str(kwargs.items())}")

                result = cache.get(cache_key)
                if result is not None:
                    return result

                result = await executor(*args, **kwargs)

                cache.set(cache_key, result)

                return result

            return wrapper

        def additional_process(func):

            def executor(*args, **kwargs):

                if pre_compute is not None:
                    args, kwargs = pre_compute(*args, **kwargs)
                if asyncio.iscoroutinefunction(func):
                    result = func(*args, **kwargs)
                else:
                    result = func(*args, **kwargs)
                if post_compute is not None:
                    result = post_compute(result)
                if row:
                    return result
                if not isinstance(result, Result):
                    result = Result.ok(data=result)
                if result.origin is None:
                    result.set_origin((mod_name if mod_name else func.__module__.split('.')[-1]
                                       , name if name else func.__name__
                                       , type_))
                if result.result.data_to == ToolBoxInterfaces.native.name:
                    result.result.data_to = ToolBoxInterfaces.remote if api else ToolBoxInterfaces.native
                # Wenden Sie die to_api_result Methode auf das Ergebnis an, falls verfügbar
                if api and hasattr(result, 'to_api_result'):
                    return result.to_api_result()
                return result

            @wraps(func)
            def wrapper(*args, **kwargs):

                if not use_cache:
                    return executor(*args, **kwargs)

                try:
                    cache_key = (f"{mod_name if mod_name else func.__module__.split('.')[-1]}"
                                 f"-{func.__name__}-{str(args)},{str(kwargs.items())}")
                except ValueError:
                    cache_key = (f"{mod_name if mod_name else func.__module__.split('.')[-1]}"
                                 f"-{func.__name__}-{bytes(args)},{str(kwargs.items())}")

                result = cache.get(cache_key)
                if result is not None:
                    return result

                result = executor(*args, **kwargs)

                cache.set(cache_key, result)

                return result

            return wrapper

        def decorator(func):
            sig = signature(func)
            params = list(sig.parameters)
            module_name = mod_name if mod_name else func.__module__.split('.')[-1]
            func_name = name if name else func.__name__
            if func_name == 'on_start':
                func_name = 'on_startup'
            if func_name == 'on_exit':
                func_name = 'on_close'
            if api or pre_compute is not None or post_compute is not None or memory_cache or file_cache:
                if asyncio.iscoroutinefunction(func):
                    func = a_additional_process(func)
                else:
                    func = additional_process(func)
            if api and str(sig.return_annotation) == 'Result':
                raise ValueError(f"Fuction {module_name}.{func_name} registered as "
                                 f"Api fuction but uses {str(sig.return_annotation)}\n"
                                 f"Please change the sig from ..)-> Result to ..)-> ApiResult")
            data = {
                "type": type_,
                "module_name": module_name,
                "func_name": func_name,
                "level": level,
                "restrict_in_virtual_mode": restrict_in_virtual_mode,
                "func": func,
                "api": api,
                "helper": helper,
                "version": version,
                "initial": initial,
                "exit_f": exit_f,
                "api_methods": api_methods if api_methods is not None else ["AUTO"],
                "__module__": func.__module__,
                "signature": sig,
                "params": params,
                "row": row,
                "state": (
                    False if len(params) == 0 else params[0] in ['self', 'state', 'app']) if state is None else state,
                "do_test": test,
                "samples": samples,
                "request_as_kwarg": request_as_kwarg,

            }

            if websocket_handler:
                # Die dekorierte Funktion sollte ein Dict mit den Handlern zurückgeben
                try:
                    handler_config = func(self)  # Rufe die Funktion auf, um die Konfiguration zu erhalten
                    if not isinstance(handler_config, dict):
                        raise TypeError(
                            f"WebSocket handler function '{func.__name__}' must return a dictionary of handlers.")

                    # Handler-Identifikator, z.B. "ChatModule/room_chat"
                    handler_id = f"{module_name}/{websocket_handler}"
                    self.websocket_handlers[handler_id] = {}

                    for event_name, handler_func in handler_config.items():
                        if event_name in ["on_connect", "on_message", "on_disconnect"] and callable(handler_func):
                            self.websocket_handlers[handler_id][event_name] = handler_func
                        else:
                            self.logger.warning(f"Invalid WebSocket handler event '{event_name}' in '{handler_id}'.")

                    self.logger.info(f"Registered WebSocket handlers for '{handler_id}'.")

                except Exception as e:
                    self.logger.error(f"Failed to register WebSocket handlers for '{func.__name__}': {e}",
                                      exc_info=True)
            else:
                self._register_function(module_name, func_name, data)

            if exit_f:
                if "on_exit" not in self.functions[module_name]:
                    self.functions[module_name]["on_exit"] = []
                self.functions[module_name]["on_exit"].append(func_name)
            if initial:
                if "on_start" not in self.functions[module_name]:
                    self.functions[module_name]["on_start"] = []
                self.functions[module_name]["on_start"].append(func_name)

            return func

        decorator.tb_init = True

        return decorator

    def export(self, *args, **kwargs):
        return self.tb(*args, **kwargs)

    def tb(self, name=None,
           mod_name: str = "",
           helper: str = "",
           version: str | None = None,
           test: bool = True,
           restrict_in_virtual_mode: bool = False,
           api: bool = False,
           initial: bool = False,
           exit_f: bool = False,
           test_only: bool = False,
           memory_cache: bool = False,
           file_cache: bool = False,
           request_as_kwarg: bool = False,
           row: bool = False,
           state: bool | None = None,
           level: int = -1,
           memory_cache_max_size: int = 100,
           memory_cache_ttl: int = 300,
           samples: list or dict or None = None,
           interface: ToolBoxInterfaces or None or str = None,
           pre_compute=None,
           post_compute=None,
           api_methods=None,
           websocket_handler: str | None = None,
           ):
        """
    A decorator for registering and configuring functions within a module.

    This decorator is used to wrap functions with additional functionality such as caching, API conversion, and lifecycle management (initialization and exit). It also handles the registration of the function in the module's function registry.

    Args:
        name (str, optional): The name to register the function under. Defaults to the function's own name.
        mod_name (str, optional): The name of the module the function belongs to.
        helper (str, optional): A helper string providing additional information about the function.
        version (str or None, optional): The version of the function or module.
        test (bool, optional): Flag to indicate if the function is for testing purposes.
        restrict_in_virtual_mode (bool, optional): Flag to restrict the function in virtual mode.
        api (bool, optional): Flag to indicate if the function is part of an API.
        initial (bool, optional): Flag to indicate if the function should be executed at initialization.
        exit_f (bool, optional): Flag to indicate if the function should be executed at exit.
        test_only (bool, optional): Flag to indicate if the function should only be used for testing.
        memory_cache (bool, optional): Flag to enable memory caching for the function.
        request_as_kwarg (bool, optional): Flag to get request if the fuction is calld from api.
        file_cache (bool, optional): Flag to enable file caching for the function.
        row (bool, optional): rather to auto wrap the result in Result type default False means no row data aka result type
        state (bool or None, optional): Flag to indicate if the function maintains state.
        level (int, optional): The level of the function, used for prioritization or categorization.
        memory_cache_max_size (int, optional): Maximum size of the memory cache.
        memory_cache_ttl (int, optional): Time-to-live for the memory cache entries.
        samples (list or dict or None, optional): Samples or examples of function usage.
        interface (str, optional): The interface type for the function.
        pre_compute (callable, optional): A function to be called before the main function.
        post_compute (callable, optional): A function to be called after the main function.
        api_methods (list[str], optional): default ["AUTO"] (GET if not params, POST if params) , GET, POST, PUT or DELETE.
        websocket_handler (str, optional): The name of the websocket handler to use.

    Returns:
        function: The decorated function with additional processing and registration capabilities.
    """
        if interface is None:
            interface = "tb"
        if test_only and 'test' not in self.id:
            return lambda *args, **kwargs: args
        return self._create_decorator(interface,
                                      name,
                                      mod_name,
                                      level=level,
                                      restrict_in_virtual_mode=restrict_in_virtual_mode,
                                      helper=helper,
                                      api=api,
                                      version=version,
                                      initial=initial,
                                      exit_f=exit_f,
                                      test=test,
                                      samples=samples,
                                      state=state,
                                      pre_compute=pre_compute,
                                      post_compute=post_compute,
                                      memory_cache=memory_cache,
                                      file_cache=file_cache,
                                      request_as_kwarg=request_as_kwarg,
                                      row=row,
                                      api_methods=api_methods,
                                      memory_cache_max_size=memory_cache_max_size,
                                      memory_cache_ttl=memory_cache_ttl,
                                      websocket_handler=websocket_handler,
                                      )

    def save_autocompletion_dict(self):
        autocompletion_dict = {}
        for module_name, _module in self.functions.items():
            data = {}
            for function_name, function_data in self.functions[module_name].items():
                if not isinstance(function_data, dict):
                    continue
                data[function_name] = {arg: None for arg in
                                       function_data.get("params", [])}
                if len(data[function_name].keys()) == 0:
                    data[function_name] = None
            autocompletion_dict[module_name] = data if len(data.keys()) > 0 else None
        self.config_fh.add_to_save_file_handler("auto~~~~~~", str(autocompletion_dict))

    def get_autocompletion_dict(self):
        return self.config_fh.get_file_handler("auto~~~~~~")

    def save_registry_as_enums(self, directory: str, filename: str):
        # Ordner erstellen, falls nicht vorhanden
        if not os.path.exists(directory):
            os.makedirs(directory)

        # Dateipfad vorbereiten
        filepath = os.path.join(directory, filename)

        # Enum-Klassen als Strings generieren
        enum_classes = [f'"""Automatic generated by ToolBox v = {self.version}"""'
                        f'\nfrom enum import Enum\nfrom dataclasses import dataclass'
                        f'\n\n\n']
        for module, functions in self.functions.items():
            if module.startswith("APP_INSTANCE"):
                continue
            class_name = module
            enum_members = "\n    ".join(
                [
                    f"{func_name.upper().replace('-', '')}"
                    f" = '{func_name}' "
                    f"# Input: ({fuction_data['params'] if isinstance(fuction_data, dict) else ''}),"
                    f" Output: {fuction_data['signature'].return_annotation if isinstance(fuction_data, dict) else 'None'}"
                    for func_name, fuction_data in functions.items()])
            enum_class = (f'@dataclass\nclass {class_name.upper().replace(".", "_").replace("-", "")}(Enum):'
                          f"\n    NAME = '{class_name}'\n    {enum_members}")
            enum_classes.append(enum_class)

        # Enums in die Datei schreiben
        data = "\n\n\n".join(enum_classes)
        if len(data) < 12:
            raise ValueError(
                "Invalid Enums Loosing content pleas delete it ur self in the (utils/system/all_functions_enums.py) or add mor new stuff :}")
        with open(filepath, 'w') as file:
            file.write(data)

        print(Style.Bold(Style.BLUE(f"Enums gespeichert in {filepath}")))


    # WS logic

    def _set_rust_ws_bridge(self, bridge_object: Any):
        """
        Diese Methode wird von Rust aufgerufen, um die Kommunikationsbrücke zu setzen.
        Sie darf NICHT manuell von Python aus aufgerufen werden.
        """
        self.print(f"Rust WebSocket bridge has been set for instance {self.id}.")
        self._rust_ws_bridge = bridge_object

    async def ws_send(self, conn_id: str, payload: dict):
        """
        Sendet eine Nachricht asynchron an eine einzelne WebSocket-Verbindung.

        Args:
            conn_id: Die eindeutige ID der Zielverbindung.
            payload: Ein Dictionary, das als JSON gesendet wird.
        """
        if self._rust_ws_bridge is None:
            self.logger.error("Cannot send WebSocket message: Rust bridge is not initialized.")
            return

        try:
            # Ruft die asynchrone Rust-Methode auf und wartet auf deren Abschluss
            await self._rust_ws_bridge.send_message(conn_id, json.dumps(payload))
        except Exception as e:
            self.logger.error(f"Failed to send WebSocket message to {conn_id}: {e}", exc_info=True)

    async def ws_broadcast(self, channel_id: str, payload: dict, source_conn_id: str = "python_broadcast"):
        """
        Sendet eine Nachricht asynchron an alle Clients in einem Kanal/Raum.

        Args:
            channel_id: Der Kanal, an den gesendet werden soll.
            payload: Ein Dictionary, das als JSON gesendet wird.
            source_conn_id (optional): Die ID der ursprünglichen Verbindung, um Echos zu vermeiden.
        """
        if self._rust_ws_bridge is None:
            self.logger.error("Cannot broadcast WebSocket message: Rust bridge is not initialized.")
            return

        try:
            # Ruft die asynchrone Rust-Broadcast-Methode auf
            await self._rust_ws_bridge.broadcast_message(channel_id, json.dumps(payload), source_conn_id)
        except Exception as e:
            self.logger.error(f"Failed to broadcast WebSocket message to channel {channel_id}: {e}", exc_info=True)

disconnect(*args, **kwargs) staticmethod

proxi attr

Source code in toolboxv2/utils/toolbox.py

@staticmethod
def disconnect(*args, **kwargs):
    """proxi attr"""

exit_main(*args, **kwargs) staticmethod

proxi attr

Source code in toolboxv2/utils/toolbox.py

@staticmethod
def exit_main(*args, **kwargs):
    """proxi attr"""

get_function(name, **kwargs)

Kwargs for _get_function metadata:: return the registered function dictionary stateless: (function_data, None), 0 stateful: (function_data, higher_order_function), 0 state::boolean specification::str default app

Source code in toolboxv2/utils/toolbox.py

def get_function(self, name: Enum or tuple, **kwargs):
    """
    Kwargs for _get_function
        metadata:: return the registered function dictionary
            stateless: (function_data, None), 0
            stateful: (function_data, higher_order_function), 0
        state::boolean
            specification::str default app
    """
    if isinstance(name, tuple):
        return self._get_function(None, as_str=name, **kwargs)
    else:
        return self._get_function(name, **kwargs)

hide_console(*args, **kwargs) staticmethod

proxi attr

Source code in toolboxv2/utils/toolbox.py

@staticmethod
def hide_console(*args, **kwargs):
    """proxi attr"""

init_mod(mod_name, spec='app')

Initializes a module in a thread-safe manner by submitting the asynchronous initialization to the running event loop.

Source code in toolboxv2/utils/toolbox.py

def init_mod(self, mod_name, spec='app'):
    """
    Initializes a module in a thread-safe manner by submitting the
    asynchronous initialization to the running event loop.
    """
    if '.' in mod_name:
        mod_name = mod_name.split('.')[0]
    self.run_bg_task(self.a_init_mod, mod_name, spec)

run(*args, request=None, running_function_coro=None, **kwargs)

Run a function with support for SSE streaming in both threaded and non-threaded contexts.

Source code in toolboxv2/utils/toolbox.py

def run(self, *args, request=None, running_function_coro=None, **kwargs):
    """
    Run a function with support for SSE streaming in both
    threaded and non-threaded contexts.
    """
    if running_function_coro is None:
        mn, fn = args[0]
        if self.functions.get(mn, {}).get(fn, {}).get('request_as_kwarg', False):
            kwargs["request"] = RequestData.from_dict(request)
            if 'data' in kwargs and 'data' not in self.functions.get(mn, {}).get(fn, {}).get('params', []):
                kwargs["request"].data = kwargs["request"].body = kwargs['data']
                del kwargs['data']
            if 'form_data' in kwargs and 'form_data' not in self.functions.get(mn, {}).get(fn, {}).get('params',
                                                                                                       []):
                kwargs["request"].form_data = kwargs["request"].body = kwargs['form_data']
                del kwargs['form_data']
        else:
            params = self.functions.get(mn, {}).get(fn, {}).get('params', [])
            # auto pars data and form_data to kwargs by key
            do = False
            data = {}
            if 'data' in kwargs and 'data' not in params:
                do = True
                data = kwargs['data']
                del kwargs['data']
            if 'form_data' in kwargs and 'form_data' not in params:
                do = True
                data = kwargs['form_data']
                del kwargs['form_data']
            if do:
                for k in params:
                    if k in data:
                        kwargs[k] = data[k]
                        del data[k]

    # Create the coroutine
    coro = running_function_coro or self.a_run_any(*args, **kwargs)

    # Get or create an event loop
    try:
        loop = asyncio.get_event_loop()
        is_running = loop.is_running()
    except RuntimeError:
        loop = asyncio.new_event_loop()
        asyncio.set_event_loop(loop)
        is_running = False

    # If the loop is already running, run in a separate thread
    if is_running:
        # Create thread pool executor as needed
        if not hasattr(self.__class__, '_executor'):
            self.__class__._executor = ThreadPoolExecutor(max_workers=4)

        def run_in_new_thread():
            # Set up a new loop in this thread
            new_loop = asyncio.new_event_loop()
            asyncio.set_event_loop(new_loop)

            try:
                # Run the coroutine
                return new_loop.run_until_complete(coro)
            finally:
                new_loop.close()

        # Run in thread and get result
        thread_result = self.__class__._executor.submit(run_in_new_thread).result()

        # Handle streaming results from thread
        if isinstance(thread_result, dict) and thread_result.get("is_stream"):
            # Create a new SSE stream in the main thread
            async def stream_from_function():
                # Re-run the function with direct async access
                stream_result = await self.a_run_any(*args, **kwargs)

                if (isinstance(stream_result, Result) and
                    getattr(stream_result.result, 'data_type', None) == "stream"):
                    # Get and forward data from the original generator
                    original_gen = stream_result.result.data.get("generator")
                    if inspect.isasyncgen(original_gen):
                        async for item in original_gen:
                            yield item

            # Return a new streaming Result
            return Result.stream(
                stream_generator=stream_from_function(),
                headers=thread_result.get("headers", {})
            )

        result = thread_result
    else:
        # Direct execution when loop is not running
        result = loop.run_until_complete(coro)

    # Process the final result
    if isinstance(result, Result):
        if 'debug' in self.id:
            result.print()
        if getattr(result.result, 'data_type', None) == "stream":
            return result
        return result.to_api_result().model_dump(mode='json')

    return result

run_bg_task(task, *args, **kwargs)

Runs a coroutine in the background without blocking the caller.

This is the primary method for "fire-and-forget" async tasks. It schedules the coroutine to run on the application's main event loop.

Parameters:

Name	Type	Description	Default
task	Callable	The coroutine function to run.	required
*args		Arguments to pass to the coroutine function.	()
**kwargs		Keyword arguments to pass to the coroutine function.	{}

Returns:

Type	Description
Task | None	An asyncio.Task object representing the scheduled task, or None if
Task | None	the task could not be scheduled.

Source code in toolboxv2/utils/toolbox.py

def run_bg_task(self, task: Callable, *args, **kwargs) -> asyncio.Task | None:
    """
    Runs a coroutine in the background without blocking the caller.

    This is the primary method for "fire-and-forget" async tasks. It schedules
    the coroutine to run on the application's main event loop.

    Args:
        task: The coroutine function to run.
        *args: Arguments to pass to the coroutine function.
        **kwargs: Keyword arguments to pass to the coroutine function.

    Returns:
        An asyncio.Task object representing the scheduled task, or None if
        the task could not be scheduled.
    """
    if not callable(task):
        self.logger.warning("Task passed to run_bg_task is not callable!")
        return None

    if not asyncio.iscoroutinefunction(task) and not asyncio.iscoroutine(task):
        self.logger.warning(f"Task '{getattr(task, '__name__', 'unknown')}' is not a coroutine. "
                            f"Use run_bg_task_advanced for synchronous functions.")
        # Fallback to advanced runner for convenience
        self.run_bg_task_advanced(task, *args, **kwargs)
        return None

    try:
        loop = self.loop_gard()
        if not loop.is_running():
            # If the main loop isn't running, we can't create a task on it.
            # This scenario is handled by run_bg_task_advanced.
            self.logger.info("Main event loop not running. Delegating to advanced background runner.")
            return self.run_bg_task_advanced(task, *args, **kwargs)

        # Create the coroutine if it's a function
        coro = task(*args, **kwargs) if asyncio.iscoroutinefunction(task) else task

        # Create a task on the running event loop
        bg_task = loop.create_task(coro)

        # Add a callback to log exceptions from the background task
        def _log_exception(the_task: asyncio.Task):
            if not the_task.cancelled() and the_task.exception():
                self.logger.error(f"Exception in background task '{the_task.get_name()}':",
                                  exc_info=the_task.exception())

        bg_task.add_done_callback(_log_exception)
        self.bg_tasks.append(bg_task)
        return bg_task

    except Exception as e:
        self.logger.error(f"Failed to schedule background task: {e}", exc_info=True)
        return None

run_bg_task_advanced(task, *args, **kwargs)

Runs a task in a separate, dedicated background thread with its own event loop.

This is ideal for: 1. Running an async task from a synchronous context. 2. Launching a long-running, independent operation that should not interfere with the main application's event loop.

Parameters:

Name	Type	Description	Default
task	Callable	The function to run (can be sync or async).	required
*args		Arguments for the task.	()
**kwargs		Keyword arguments for the task.	{}

Returns:

Type	Description
Thread	The threading.Thread object managing the background execution.

Source code in toolboxv2/utils/toolbox.py

def run_bg_task_advanced(self, task: Callable, *args, **kwargs) -> threading.Thread:
    """
    Runs a task in a separate, dedicated background thread with its own event loop.

    This is ideal for:
    1. Running an async task from a synchronous context.
    2. Launching a long-running, independent operation that should not
       interfere with the main application's event loop.

    Args:
        task: The function to run (can be sync or async).
        *args: Arguments for the task.
        **kwargs: Keyword arguments for the task.

    Returns:
        The threading.Thread object managing the background execution.
    """
    if not callable(task):
        self.logger.warning("Task for run_bg_task_advanced is not callable!")
        return None

    def thread_target():
        # Each thread gets its own event loop.
        loop = asyncio.new_event_loop()
        asyncio.set_event_loop(loop)

        try:
            # Prepare the coroutine we need to run
            if asyncio.iscoroutinefunction(task):
                coro = task(*args, **kwargs)
            elif asyncio.iscoroutine(task):
                # It's already a coroutine object
                coro = task
            else:
                # It's a synchronous function, run it in an executor
                # to avoid blocking the new event loop.
                coro = loop.run_in_executor(None, lambda: task(*args, **kwargs))

            # Run the coroutine to completion
            result = loop.run_until_complete(coro)
            self.logger.debug(f"Advanced background task '{getattr(task, '__name__', 'unknown')}' completed.")
            if result is not None:
                self.logger.debug(f"Task result: {str(result)[:100]}")

        except Exception as e:
            self.logger.error(f"Error in advanced background task '{getattr(task, '__name__', 'unknown')}':",
                              exc_info=e)
        finally:
            # Cleanly shut down the event loop in this thread.
            try:
                all_tasks = asyncio.all_tasks(loop=loop)
                if all_tasks:
                    for t in all_tasks:
                        t.cancel()
                    loop.run_until_complete(asyncio.gather(*all_tasks, return_exceptions=True))
            finally:
                loop.close()
                asyncio.set_event_loop(None)

    # Create, start, and return the thread.
    # It's a daemon thread so it won't prevent the main app from exiting.
    t = threading.Thread(target=thread_target, daemon=True, name=f"BGTask-{getattr(task, '__name__', 'unknown')}")
    self.bg_tasks.append(t)
    t.start()
    return t

show_console(*args, **kwargs) staticmethod

proxi attr

Source code in toolboxv2/utils/toolbox.py

@staticmethod
def show_console(*args, **kwargs):
    """proxi attr"""

tb(name=None, mod_name='', helper='', version=None, test=True, restrict_in_virtual_mode=False, api=False, initial=False, exit_f=False, test_only=False, memory_cache=False, file_cache=False, request_as_kwarg=False, row=False, state=None, level=-1, memory_cache_max_size=100, memory_cache_ttl=300, samples=None, interface=None, pre_compute=None, post_compute=None, api_methods=None, websocket_handler=None)

A decorator for registering and configuring functions within a module.

This decorator is used to wrap functions with additional functionality such as caching, API conversion, and lifecycle management (initialization and exit). It also handles the registration of the function in the module's function registry.

Parameters:

Name	Type	Description	Default
name	str	The name to register the function under. Defaults to the function's own name.	None
mod_name	str	The name of the module the function belongs to.	''
helper	str	A helper string providing additional information about the function.	''
version	str or None	The version of the function or module.	None
test	bool	Flag to indicate if the function is for testing purposes.	True
restrict_in_virtual_mode	bool	Flag to restrict the function in virtual mode.	False
api	bool	Flag to indicate if the function is part of an API.	False
initial	bool	Flag to indicate if the function should be executed at initialization.	False
exit_f	bool	Flag to indicate if the function should be executed at exit.	False
test_only	bool	Flag to indicate if the function should only be used for testing.	False
memory_cache	bool	Flag to enable memory caching for the function.	False
request_as_kwarg	bool	Flag to get request if the fuction is calld from api.	False
file_cache	bool	Flag to enable file caching for the function.	False
row	bool	rather to auto wrap the result in Result type default False means no row data aka result type	False
state	bool or None	Flag to indicate if the function maintains state.	None
level	int	The level of the function, used for prioritization or categorization.	-1
memory_cache_max_size	int	Maximum size of the memory cache.	100
memory_cache_ttl	int	Time-to-live for the memory cache entries.	300
samples	list or dict or None	Samples or examples of function usage.	None
interface	str	The interface type for the function.	None
pre_compute	callable	A function to be called before the main function.	None
post_compute	callable	A function to be called after the main function.	None
api_methods	list[str]	default ["AUTO"] (GET if not params, POST if params) , GET, POST, PUT or DELETE.	None
websocket_handler	str	The name of the websocket handler to use.	None

Returns:

Name	Type	Description
function		The decorated function with additional processing and registration capabilities.

Source code in toolboxv2/utils/toolbox.py

def tb(self, name=None,
       mod_name: str = "",
       helper: str = "",
       version: str | None = None,
       test: bool = True,
       restrict_in_virtual_mode: bool = False,
       api: bool = False,
       initial: bool = False,
       exit_f: bool = False,
       test_only: bool = False,
       memory_cache: bool = False,
       file_cache: bool = False,
       request_as_kwarg: bool = False,
       row: bool = False,
       state: bool | None = None,
       level: int = -1,
       memory_cache_max_size: int = 100,
       memory_cache_ttl: int = 300,
       samples: list or dict or None = None,
       interface: ToolBoxInterfaces or None or str = None,
       pre_compute=None,
       post_compute=None,
       api_methods=None,
       websocket_handler: str | None = None,
       ):
    """
A decorator for registering and configuring functions within a module.

This decorator is used to wrap functions with additional functionality such as caching, API conversion, and lifecycle management (initialization and exit). It also handles the registration of the function in the module's function registry.

Args:
    name (str, optional): The name to register the function under. Defaults to the function's own name.
    mod_name (str, optional): The name of the module the function belongs to.
    helper (str, optional): A helper string providing additional information about the function.
    version (str or None, optional): The version of the function or module.
    test (bool, optional): Flag to indicate if the function is for testing purposes.
    restrict_in_virtual_mode (bool, optional): Flag to restrict the function in virtual mode.
    api (bool, optional): Flag to indicate if the function is part of an API.
    initial (bool, optional): Flag to indicate if the function should be executed at initialization.
    exit_f (bool, optional): Flag to indicate if the function should be executed at exit.
    test_only (bool, optional): Flag to indicate if the function should only be used for testing.
    memory_cache (bool, optional): Flag to enable memory caching for the function.
    request_as_kwarg (bool, optional): Flag to get request if the fuction is calld from api.
    file_cache (bool, optional): Flag to enable file caching for the function.
    row (bool, optional): rather to auto wrap the result in Result type default False means no row data aka result type
    state (bool or None, optional): Flag to indicate if the function maintains state.
    level (int, optional): The level of the function, used for prioritization or categorization.
    memory_cache_max_size (int, optional): Maximum size of the memory cache.
    memory_cache_ttl (int, optional): Time-to-live for the memory cache entries.
    samples (list or dict or None, optional): Samples or examples of function usage.
    interface (str, optional): The interface type for the function.
    pre_compute (callable, optional): A function to be called before the main function.
    post_compute (callable, optional): A function to be called after the main function.
    api_methods (list[str], optional): default ["AUTO"] (GET if not params, POST if params) , GET, POST, PUT or DELETE.
    websocket_handler (str, optional): The name of the websocket handler to use.

Returns:
    function: The decorated function with additional processing and registration capabilities.
"""
    if interface is None:
        interface = "tb"
    if test_only and 'test' not in self.id:
        return lambda *args, **kwargs: args
    return self._create_decorator(interface,
                                  name,
                                  mod_name,
                                  level=level,
                                  restrict_in_virtual_mode=restrict_in_virtual_mode,
                                  helper=helper,
                                  api=api,
                                  version=version,
                                  initial=initial,
                                  exit_f=exit_f,
                                  test=test,
                                  samples=samples,
                                  state=state,
                                  pre_compute=pre_compute,
                                  post_compute=post_compute,
                                  memory_cache=memory_cache,
                                  file_cache=file_cache,
                                  request_as_kwarg=request_as_kwarg,
                                  row=row,
                                  api_methods=api_methods,
                                  memory_cache_max_size=memory_cache_max_size,
                                  memory_cache_ttl=memory_cache_ttl,
                                  websocket_handler=websocket_handler,
                                  )

wait_for_bg_tasks(timeout=None)

Wait for all background tasks to complete.

Parameters:

Name	Type	Description	Default
timeout		Maximum time to wait (in seconds) for all tasks to complete. None means wait indefinitely.	None

Returns:

Name	Type	Description
bool		True if all tasks completed, False if timeout occurred

Source code in toolboxv2/utils/toolbox.py

def wait_for_bg_tasks(self, timeout=None):
    """
    Wait for all background tasks to complete.

    Args:
        timeout: Maximum time to wait (in seconds) for all tasks to complete.
                 None means wait indefinitely.

    Returns:
        bool: True if all tasks completed, False if timeout occurred
    """
    active_tasks = [t for t in self.bg_tasks if t.is_alive()]

    for task in active_tasks:
        task.join(timeout=timeout)
        if task.is_alive():
            return False

    return True

ws_broadcast(channel_id, payload, source_conn_id='python_broadcast') async

Sendet eine Nachricht asynchron an alle Clients in einem Kanal/Raum.

Parameters:

Name	Type	Description	Default
channel_id	str	Der Kanal, an den gesendet werden soll.	required
payload	dict	Ein Dictionary, das als JSON gesendet wird.	required
source_conn_id	optional	Die ID der ursprünglichen Verbindung, um Echos zu vermeiden.	'python_broadcast'

Source code in toolboxv2/utils/toolbox.py

async def ws_broadcast(self, channel_id: str, payload: dict, source_conn_id: str = "python_broadcast"):
    """
    Sendet eine Nachricht asynchron an alle Clients in einem Kanal/Raum.

    Args:
        channel_id: Der Kanal, an den gesendet werden soll.
        payload: Ein Dictionary, das als JSON gesendet wird.
        source_conn_id (optional): Die ID der ursprünglichen Verbindung, um Echos zu vermeiden.
    """
    if self._rust_ws_bridge is None:
        self.logger.error("Cannot broadcast WebSocket message: Rust bridge is not initialized.")
        return

    try:
        # Ruft die asynchrone Rust-Broadcast-Methode auf
        await self._rust_ws_bridge.broadcast_message(channel_id, json.dumps(payload), source_conn_id)
    except Exception as e:
        self.logger.error(f"Failed to broadcast WebSocket message to channel {channel_id}: {e}", exc_info=True)

ws_send(conn_id, payload) async

Sendet eine Nachricht asynchron an eine einzelne WebSocket-Verbindung.

Parameters:

Name	Type	Description	Default
conn_id	str	Die eindeutige ID der Zielverbindung.	required
payload	dict	Ein Dictionary, das als JSON gesendet wird.	required

Source code in toolboxv2/utils/toolbox.py

async def ws_send(self, conn_id: str, payload: dict):
    """
    Sendet eine Nachricht asynchron an eine einzelne WebSocket-Verbindung.

    Args:
        conn_id: Die eindeutige ID der Zielverbindung.
        payload: Ein Dictionary, das als JSON gesendet wird.
    """
    if self._rust_ws_bridge is None:
        self.logger.error("Cannot send WebSocket message: Rust bridge is not initialized.")
        return

    try:
        # Ruft die asynchrone Rust-Methode auf und wartet auf deren Abschluss
        await self._rust_ws_bridge.send_message(conn_id, json.dumps(payload))
    except Exception as e:
        self.logger.error(f"Failed to send WebSocket message to {conn_id}: {e}", exc_info=True)

toolboxv2.show_console(show=True)

Source code in toolboxv2/utils/extras/show_and_hide_console.py

def show_console(show=True):
    global TBRUNNER_console_viabel
    """Brings up the Console Window."""
    try:
        if show and not TBRUNNER_console_viabel:
            # Show console
            ctypes.windll.user32.ShowWindow(ctypes.windll.kernel32.GetConsoleWindow(), 4)
            TBRUNNER_console_viabel = True
            return True
        elif not show and TBRUNNER_console_viabel:
            # Hide console
            ctypes.windll.user32.ShowWindow(ctypes.windll.kernel32.GetConsoleWindow(), 0)
            TBRUNNER_console_viabel = False
            return True
    except:
        print(f"Could not show_console {show=}", )
        return False
    return False

Logging

toolboxv2.get_logger()

Source code in toolboxv2/utils/system/tb_logger.py

def get_logger() -> logging.Logger:
    return logging.getLogger(loggerNameOfToolboxv2)

toolboxv2.setup_logging(level, name=loggerNameOfToolboxv2, online_level=None, is_online=False, file_level=None, interminal=False, logs_directory='../logs', app_name='main')

Source code in toolboxv2/utils/system/tb_logger.py

def setup_logging(level: int, name=loggerNameOfToolboxv2, online_level=None, is_online=False, file_level=None,
                  interminal=False, logs_directory="../logs", app_name="main"):
    global loggerNameOfToolboxv2

    if not online_level:
        online_level = level

    if not file_level:
        file_level = level

    if not os.path.exists(logs_directory):
        os.makedirs(logs_directory, exist_ok=True)
    if not os.path.exists(logs_directory + "/Logs.info"):
        open(f"{logs_directory}/Logs.info", "a").close()

    loggerNameOfToolboxv2 = name

    available_log_levels = [logging.CRITICAL, logging.FATAL, logging.ERROR, logging.WARNING, logging.WARN, logging.INFO,
                            logging.DEBUG, logging.NOTSET]

    if level not in available_log_levels:
        raise ValueError(f"level must be one of {available_log_levels}, but logging level is {level}")

    if online_level not in available_log_levels:
        raise ValueError(f"online_level must be one of {available_log_levels}, but logging level is {online_level}")

    if file_level not in available_log_levels:
        raise ValueError(f"file_level must be one of {available_log_levels}, but logging level is {file_level}")

    log_date = datetime.datetime.today().strftime('%Y-%m-%d')
    log_levels = ["CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG", "NOTSET"]
    log_level_index = log_levels.index(logging.getLevelName(level))

    filename = f"Logs-{name}-{log_date}-{log_levels[log_level_index]}"
    log_filename = f"{logs_directory}/{filename}.log"

    log_info_data = {
        filename: 0,
        "H": "localhost",
        "P": 62435
    }

    with open(f"{logs_directory}/Logs.info") as li:
        log_info_data_str = li.read()
        try:
            log_info_data = eval(log_info_data_str)
        except SyntaxError:
            if log_info_data_str:
                print(Style.RED(Style.Bold("Could not parse log info data")))

        if filename not in log_info_data:
            log_info_data[filename] = 0

        if not os.path.exists(log_filename):
            log_info_data[filename] = 0
            print("new log file")

        if os.path.exists(log_filename):
            log_info_data[filename] += 1

            while os.path.exists(f"{logs_directory}/{filename}#{log_info_data[filename]}.log"):
                log_info_data[filename] += 1

            try:
                os.rename(log_filename,
                          f"{logs_directory}/{filename}#{log_info_data[filename]}.log")
            except PermissionError:
                print(Style.YELLOW(Style.Bold(f"Could not rename log file appending on {filename}")))

    with open(f"{logs_directory}/Logs.info", "w") as li:
        if len(log_info_data.keys()) >= 7:
            log_info_data = {
                filename: log_info_data[filename],
                "H": log_info_data["H"],
                "P": log_info_data["P"]
            }
        li.write(str(log_info_data))

    try:
        with open(log_filename, "a"):
            pass
    except OSError:
        log_filename = f"{logs_directory}/Logs-Test-{log_date}-{log_levels[log_level_index]}.log"
        with open(log_filename, "a"):
            pass

    logger = logging.getLogger(name)

    logger.setLevel(level)
    # Prevent logger from propagating to parent loggers
    logger.propagate = False

    terminal_format = f"{app_name} %(asctime)s %(levelname)s %(name)s - %(message)s"
    file_format = f"{app_name} %(asctime)s - %(name)s - %(levelname)s - %(filename)s - %(funcName)s:%(lineno)d - %(message)s"

    # Configure handlers
    handlers = []

    # File handler (always added)
    file_handler = logging.FileHandler(log_filename)
    file_handler.setFormatter(logging.Formatter(file_format))
    file_handler.setLevel(file_level)
    handlers.append(file_handler)

    # Terminal handler (if requested)
    if interminal:
        terminal_handler = logging.StreamHandler()
        terminal_handler.setFormatter(logging.Formatter(terminal_format))
        terminal_handler.setLevel(level)
        handlers.append(terminal_handler)

    # Socket handler (if requested)
    if is_online:
        socket_handler = SocketHandler(log_info_data["H"], log_info_data["P"])
        socket_handler.setFormatter(logging.Formatter(file_format))
        socket_handler.setLevel(online_level)
        handlers.append(socket_handler)

    # Add all handlers to logger
    for handler in handlers:
        logger.addHandler(handler)

    return logger, filename

Styling & Console Output

toolboxv2.Style

Source code in toolboxv2/utils/extras/Style.py

class Style:
    _END = '\33[0m'
    _BLACK = '\33[30m'
    _RED = '\33[31m'
    _GREEN = '\33[32m'
    _YELLOW = '\33[33m'
    _BLUE = '\33[34m'
    _MAGENTA = '\33[35m'
    _CYAN = '\33[36m'
    _WHITE = '\33[37m'

    _Bold = '\33[1m'
    _ITALIC = '\33[3m'
    _Underline = '\33[4m'
    _BLINK = '\33[5m'
    _BLINK2 = '\33[6m'
    _Reversed = '\33[7m'

    _BLACKBG = '\33[40m'
    _REDBG = '\33[41m'
    _GREENBG = '\33[42m'
    _YELLOWBG = '\33[43m'
    _BLUEBG = '\33[44m'
    _VIOLETBG = '\33[45m'
    _BEIGEBG = '\33[46m'
    _WHITEBG = '\33[47m'

    _GREY = '\33[90m'
    _RED2 = '\33[91m'
    _GREEN2 = '\33[92m'
    _YELLOW2 = '\33[93m'
    _BLUE2 = '\33[94m'
    _VIOLET2 = '\33[95m'
    _BEIGE2 = '\33[96m'
    _WHITE2 = '\33[97m'

    _GREYBG = '\33[100m'
    _REDBG2 = '\33[101m'
    _GREENBG2 = '\33[102m'
    _YELLOWBG2 = '\33[103m'
    _BLUEBG2 = '\33[104m'
    _VIOLETBG2 = '\33[105m'
    _BEIGEBG2 = '\33[106m'
    _WHITEBG2 = '\33[107m'

    style_dic = {
        "END": _END,
        "BLACK": _BLACK,
        "RED": _RED,
        "GREEN": _GREEN,
        "YELLOW": _YELLOW,
        "BLUE": _BLUE,
        "MAGENTA": _MAGENTA,
        "CYAN": _CYAN,
        "WHITE": _WHITE,
        "Bold": _Bold,
        "Underline": _Underline,
        "Reversed": _Reversed,

        "ITALIC": _ITALIC,
        "BLINK": _BLINK,
        "BLINK2": _BLINK2,
        "BLACKBG": _BLACKBG,
        "REDBG": _REDBG,
        "GREENBG": _GREENBG,
        "YELLOWBG": _YELLOWBG,
        "BLUEBG": _BLUEBG,
        "VIOLETBG": _VIOLETBG,
        "BEIGEBG": _BEIGEBG,
        "WHITEBG": _WHITEBG,
        "GREY": _GREY,
        "RED2": _RED2,
        "GREEN2": _GREEN2,
        "YELLOW2": _YELLOW2,
        "BLUE2": _BLUE2,
        "VIOLET2": _VIOLET2,
        "BEIGE2": _BEIGE2,
        "WHITE2": _WHITE2,
        "GREYBG": _GREYBG,
        "REDBG2": _REDBG2,
        "GREENBG2": _GREENBG2,
        "YELLOWBG2": _YELLOWBG2,
        "BLUEBG2": _BLUEBG2,
        "VIOLETBG2": _VIOLETBG2,
        "BEIGEBG2": _BEIGEBG2,
        "WHITEBG2": _WHITEBG2,

    }

    @staticmethod
    @text_save
    def END_():
        print(Style._END)

    @staticmethod
    @text_save
    def GREEN_():
        print(Style._GREEN)

    @staticmethod
    @text_save
    def BLUE(text: str):
        return Style._BLUE + text + Style._END

    @staticmethod
    @text_save
    def BLACK(text: str):
        return Style._BLACK + text + Style._END

    @staticmethod
    @text_save
    def RED(text: str):
        return Style._RED + text + Style._END

    @staticmethod
    @text_save
    def GREEN(text: str):
        return Style._GREEN + text + Style._END

    @staticmethod
    @text_save
    def YELLOW(text: str):
        return Style._YELLOW + text + Style._END

    @staticmethod
    @text_save
    def MAGENTA(text: str):
        return Style._MAGENTA + text + Style._END

    @staticmethod
    @text_save
    def CYAN(text: str):
        return Style._CYAN + text + Style._END

    @staticmethod
    @text_save
    def WHITE(text: str):
        return Style._WHITE + text + Style._END

    @staticmethod
    @text_save
    def Bold(text: str):
        return Style._Bold + text + Style._END

    @staticmethod
    @text_save
    def Underline(text: str):
        return Style._Underline + text + Style._END

    @staticmethod
    @text_save
    def Underlined(text: str):
        return Style._Underline + text + Style._END

    @staticmethod
    @text_save
    def Reversed(text: str):
        return Style._Reversed + text + Style._END

    @staticmethod
    @text_save
    def ITALIC(text: str):
        return Style._ITALIC + text + Style._END

    @staticmethod
    @text_save
    def BLINK(text: str):
        return Style._BLINK + text + Style._END

    @staticmethod
    @text_save
    def BLINK2(text: str):
        return Style._BLINK2 + text + Style._END

    @staticmethod
    @text_save
    def BLACKBG(text: str):
        return Style._BLACKBG + text + Style._END

    @staticmethod
    @text_save
    def REDBG(text: str):
        return Style._REDBG + text + Style._END

    @staticmethod
    @text_save
    def GREENBG(text: str):
        return Style._GREENBG + text + Style._END

    @staticmethod
    @text_save
    def YELLOWBG(text: str):
        return Style._YELLOWBG + text + Style._END

    @staticmethod
    @text_save
    def BLUEBG(text: str):
        return Style._BLUEBG + text + Style._END

    @staticmethod
    @text_save
    def VIOLETBG(text: str):
        return Style._VIOLETBG + text + Style._END

    @staticmethod
    @text_save
    def BEIGEBG(text: str):
        return Style._BEIGEBG + text + Style._END

    @staticmethod
    @text_save
    def WHITEBG(text: str):
        return Style._WHITEBG + text + Style._END

    @staticmethod
    @text_save
    def GREY(text: str):
        return Style._GREY + str(text) + Style._END

    @staticmethod
    @text_save
    def RED2(text: str):
        return Style._RED2 + text + Style._END

    @staticmethod
    @text_save
    def GREEN2(text: str):
        return Style._GREEN2 + text + Style._END

    @staticmethod
    @text_save
    def YELLOW2(text: str):
        return Style._YELLOW2 + text + Style._END

    @staticmethod
    @text_save
    def BLUE2(text: str):
        return Style._BLUE2 + text + Style._END

    @staticmethod
    @text_save
    def VIOLET2(text: str):
        return Style._VIOLET2 + text + Style._END

    @staticmethod
    @text_save
    def BEIGE2(text: str):
        return Style._BEIGE2 + text + Style._END

    @staticmethod
    @text_save
    def WHITE2(text: str):
        return Style._WHITE2 + text + Style._END

    @staticmethod
    @text_save
    def GREYBG(text: str):
        return Style._GREYBG + text + Style._END

    @staticmethod
    @text_save
    def REDBG2(text: str):
        return Style._REDBG2 + text + Style._END

    @staticmethod
    @text_save
    def GREENBG2(text: str):
        return Style._GREENBG2 + text + Style._END

    @staticmethod
    @text_save
    def YELLOWBG2(text: str):
        return Style._YELLOWBG2 + text + Style._END

    @staticmethod
    @text_save
    def BLUEBG2(text: str):
        return Style._BLUEBG2 + text + Style._END

    @staticmethod
    @text_save
    def VIOLETBG2(text: str):
        return Style._VIOLETBG2 + text + Style._END

    @staticmethod
    @text_save
    def BEIGEBG2(text: str):
        return Style._BEIGEBG2 + text + Style._END

    @staticmethod
    @text_save
    def WHITEBG2(text: str):
        return Style._WHITEBG2 + text + Style._END

    @staticmethod
    @text_save
    def loading_al(text: str):
        b = f"{text} /"
        print(b)
        sleep(0.05)
        cls()
        b = f"{text} -"
        print(b)
        sleep(0.05)
        cls()
        b = f"{text} \\"
        print(b)
        sleep(0.05)
        cls()
        b = f"{text} |"
        print(b)
        sleep(0.05)
        cls()

    @property
    def END(self):
        return self._END

    def color_demo(self):
        for color in self.style_dic:
            print(f"{color} -> {self.style_dic[color]}Effect{self._END}")

    @property
    def Underline2(self):
        return self._Underline

toolboxv2.Spinner

Enhanced Spinner with tqdm-like line rendering.

Source code in toolboxv2/utils/extras/Style.py

class Spinner:
    """
    Enhanced Spinner with tqdm-like line rendering.
    """
    SYMBOL_SETS = {
        "c": ["◐", "◓", "◑", "◒"],
        "b": ["▁", "▃", "▄", "▅", "▆", "▇", "█", "▇", "▆", "▅", "▄", "▃"],
        "d": ["⣾", "⣽", "⣻", "⢿", "⡿", "⣟", "⣯", "⣷"],
        "w": ["🌍", "🌎", "🌏"],
        "s": ["🌀   ", " 🌀  ", "  🌀 ", "   🌀", "  🌀 ", " 🌀  "],
        "+": ["+", "x"],
        "t": ["✶", "✸", "✹", "✺", "✹", "✷"]
    }

    def __init__(
        self,
        message: str = "Loading...",
        delay: float = 0.1,
        symbols=None,
        count_down: bool = False,
        time_in_s: float = 0
    ):
        """Initialize spinner with flexible configuration."""
        # Resolve symbol set.
        if isinstance(symbols, str):
            symbols = self.SYMBOL_SETS.get(symbols, None)

        # Default symbols if not provided.
        if symbols is None:
            symbols = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]

        # Test mode symbol set.
        if 'unittest' in sys.argv[0]:
            symbols = ['#', '=', '-']

        self.spinner = itertools.cycle(symbols)
        self.delay = delay
        self.message = message
        self.running = False
        self.spinner_thread = None
        self.max_t = time_in_s
        self.contd = count_down

        # Rendering management.
        self._is_primary = False
        self._start_time = 0

        # Central manager.
        self.manager = SpinnerManager()

    def _generate_render_line(self):
        """Generate the primary render line."""
        current_time = time.time()
        if self.contd:
            remaining = max(0, self.max_t - (current_time - self._start_time))
            time_display = f"{remaining:.2f}"
        else:
            time_display = f"{current_time - self._start_time:.2f}"

        symbol = next(self.spinner)
        return f"{symbol} {self.message} | {time_display}"

    def _generate_secondary_info(self):
        """Generate secondary spinner info for additional spinners."""
        return f"{self.message}"

    def __enter__(self):
        """Start the spinner."""
        self.running = True
        self._start_time = time.time()
        self.manager.register_spinner(self)
        return self

    def __exit__(self, exc_type, exc_value, exc_traceback):
        """Stop the spinner."""
        self.running = False
        self.manager.unregister_spinner(self)
        # Clear the spinner's line if it was the primary spinner.
        if self._is_primary:
            sys.stdout.write("\r\033[K")
            sys.stdout.flush()

__enter__()

Start the spinner.

Source code in toolboxv2/utils/extras/Style.py

def __enter__(self):
    """Start the spinner."""
    self.running = True
    self._start_time = time.time()
    self.manager.register_spinner(self)
    return self

__exit__(exc_type, exc_value, exc_traceback)

Stop the spinner.

Source code in toolboxv2/utils/extras/Style.py

def __exit__(self, exc_type, exc_value, exc_traceback):
    """Stop the spinner."""
    self.running = False
    self.manager.unregister_spinner(self)
    # Clear the spinner's line if it was the primary spinner.
    if self._is_primary:
        sys.stdout.write("\r\033[K")
        sys.stdout.flush()

__init__(message='Loading...', delay=0.1, symbols=None, count_down=False, time_in_s=0)

Initialize spinner with flexible configuration.

Source code in toolboxv2/utils/extras/Style.py

def __init__(
    self,
    message: str = "Loading...",
    delay: float = 0.1,
    symbols=None,
    count_down: bool = False,
    time_in_s: float = 0
):
    """Initialize spinner with flexible configuration."""
    # Resolve symbol set.
    if isinstance(symbols, str):
        symbols = self.SYMBOL_SETS.get(symbols, None)

    # Default symbols if not provided.
    if symbols is None:
        symbols = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]

    # Test mode symbol set.
    if 'unittest' in sys.argv[0]:
        symbols = ['#', '=', '-']

    self.spinner = itertools.cycle(symbols)
    self.delay = delay
    self.message = message
    self.running = False
    self.spinner_thread = None
    self.max_t = time_in_s
    self.contd = count_down

    # Rendering management.
    self._is_primary = False
    self._start_time = 0

    # Central manager.
    self.manager = SpinnerManager()

toolboxv2.remove_styles(text, infos=False)

Source code in toolboxv2/utils/extras/Style.py

def remove_styles(text: str, infos=False):
    in_ = []
    for key, style in Style.style_dic.items():
        if style in text:
            text = text.replace(style, '')
            if infos:
                in_.append([key for key, st in Style.style_dic.items() if style == st][0])
    if infos:
        if "END" in in_:
            in_.remove('END')
        return text, in_
    return text

Data Types & Structures

toolboxv2.AppArgs

Source code in toolboxv2/utils/system/types.py

class AppArgs:
    init = None
    init_file = 'init.config'
    get_version = False
    mm = False
    sm = False
    lm = False
    modi = 'cli'
    kill = False
    remote = False
    remote_direct_key = None
    background_application = False
    background_application_runner = False
    docker = False
    build = False
    install = None
    remove = None
    update = None
    name = 'main'
    port = 5000
    host = '0.0.0.0'
    load_all_mod_in_files = False
    mods_folder = 'toolboxv2.mods.'
    debug = None
    test = None
    profiler = None
    hot_reload = False
    live_application = True
    sysPrint = False
    kwargs = {}
    session = None

    def default(self):
        return self

    def set(self, name, value):
        setattr(self, name, value)
        return self

toolboxv2.Result

Bases: Generic[T]

Source code in toolboxv2/utils/system/types.py

class Result(Generic[T]):
    _task = None
    _generic_type: Optional[Type] = None

    def __init__(self,
                 error: ToolBoxError,
                 result: ToolBoxResult,
                 info: ToolBoxInfo,
                 origin: Any | None = None,
                 generic_type: Optional[Type] = None
                 ):
        self.error: ToolBoxError = error
        self.result: ToolBoxResult = result
        self.info: ToolBoxInfo = info
        self.origin = origin
        self._generic_type = generic_type

    def __class_getitem__(cls, item):
        """Enable Result[Type] syntax"""

        class TypedResult(cls):
            _generic_type = item

            def __init__(self, *args, **kwargs):
                super().__init__(*args, **kwargs)
                self._generic_type = item

        return TypedResult

    def typed_get(self, key=None, default=None) -> T:
        """Get data with type validation"""
        data = self.get(key, default)

        if self._generic_type and data is not None:
            # Validate type matches generic parameter
            if not self._validate_type(data, self._generic_type):
                from toolboxv2 import get_logger
                get_logger().warning(f"Type mismatch: expected {self._generic_type}, got {type(data)}")

        return data

    async def typed_aget(self, key=None, default=None) -> T:
        """Async get data with type validation"""
        data = await self.aget(key, default)

        if self._generic_type and data is not None:
            if not self._validate_type(data, self._generic_type):
                from toolboxv2 import get_logger
                get_logger().warning(f"Type mismatch: expected {self._generic_type}, got {type(data)}")

        return data

    def _validate_type(self, data, expected_type) -> bool:
        """Validate data matches expected type"""
        try:
            # Handle List[Type] syntax
            origin = get_origin(expected_type)
            if origin is list or origin is List:
                if not isinstance(data, list):
                    return False

                # Check list element types if specified
                args = get_args(expected_type)
                if args and data:
                    element_type = args[0]
                    return all(isinstance(item, element_type) for item in data)
                return True

            # Handle other generic types
            elif origin is not None:
                return isinstance(data, origin)

            # Handle regular types
            else:
                return isinstance(data, expected_type)

        except Exception:
            return True  # Skip validation on error

    @classmethod
    def typed_ok(cls, data: T, data_info="", info="OK", interface=ToolBoxInterfaces.native) -> 'Result[T]':
        """Create OK result with type information"""
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=0, help_text=info)
        result = ToolBoxResult(data_to=interface, data=data, data_info=data_info, data_type=type(data).__name__)

        instance = cls(error=error, info=info_obj, result=result)
        if hasattr(cls, '_generic_type'):
            instance._generic_type = cls._generic_type

        return instance

    @classmethod
    def typed_json(cls, data: T, info="OK", interface=ToolBoxInterfaces.remote, exec_code=0,
                   status_code=None) -> 'Result[T]':
        """Create JSON result with type information"""
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=status_code or exec_code, help_text=info)

        result = ToolBoxResult(
            data_to=interface,
            data=data,
            data_info="JSON response",
            data_type="json"
        )

        instance = cls(error=error, info=info_obj, result=result)
        if hasattr(cls, '_generic_type'):
            instance._generic_type = cls._generic_type

        return instance

    def cast_to(self, target_type: Type[T]) -> 'Result[T]':
        """Cast result to different type"""
        new_result = Result(
            error=self.error,
            result=self.result,
            info=self.info,
            origin=self.origin,
            generic_type=target_type
        )
        new_result._generic_type = target_type
        return new_result

    def get_type_info(self) -> Optional[Type]:
        """Get the generic type information"""
        return self._generic_type

    def is_typed(self) -> bool:
        """Check if result has type information"""
        return self._generic_type is not None

    def as_result(self):
        return self

    def as_dict(self):
        return {
            "error":self.error.value if isinstance(self.error, Enum) else self.error,
        "result" : {
            "data_to":self.result.data_to.value if isinstance(self.result.data_to, Enum) else self.result.data_to,
            "data_info":self.result.data_info,
            "data":self.result.data,
            "data_type":self.result.data_type
        } if self.result else None,
        "info" : {
            "exec_code" : self.info.exec_code,  # exec_code umwandel in http resposn codes
        "help_text" : self.info.help_text
        } if self.info else None,
        "origin" : self.origin
        }

    def set_origin(self, origin):
        if self.origin is not None:
            raise ValueError("You cannot Change the origin of a Result!")
        self.origin = origin
        return self

    def set_dir_origin(self, name, extras="assets/"):
        if self.origin is not None:
            raise ValueError("You cannot Change the origin of a Result!")
        self.origin = f"mods/{name}/{extras}"
        return self

    def is_error(self):
        if _test_is_result(self.result.data):
            return self.result.data.is_error()
        if self.error == ToolBoxError.none:
            return False
        if self.info.exec_code == 0:
            return False
        return self.info.exec_code != 200

    def is_ok(self):
        return not self.is_error()

    def is_data(self):
        return self.result.data is not None

    def to_api_result(self):
        # print(f" error={self.error}, result= {self.result}, info= {self.info}, origin= {self.origin}")
        return ApiResult(
            error=self.error.value if isinstance(self.error, Enum) else self.error,
            result=ToolBoxResultBM(
                data_to=self.result.data_to.value if isinstance(self.result.data_to, Enum) else self.result.data_to,
                data_info=self.result.data_info,
                data=self.result.data,
                data_type=self.result.data_type
            ) if self.result else None,
            info=ToolBoxInfoBM(
                exec_code=self.info.exec_code,  # exec_code umwandel in http resposn codes
                help_text=self.info.help_text
            ) if self.info else None,
            origin=self.origin
        )

    def task(self, task):
        self._task = task
        return self

    @staticmethod
    def result_from_dict(error: str, result: dict, info: dict, origin: list or None or str):
        # print(f" error={self.error}, result= {self.result}, info= {self.info}, origin= {self.origin}")
        return ApiResult(
            error=error if isinstance(error, Enum) else error,
            result=ToolBoxResultBM(
                data_to=result.get('data_to') if isinstance(result.get('data_to'), Enum) else result.get('data_to'),
                data_info=result.get('data_info', '404'),
                data=result.get('data'),
                data_type=result.get('data_type', '404'),
            ) if result else ToolBoxResultBM(
                data_to=ToolBoxInterfaces.cli.value,
                data_info='',
                data='404',
                data_type='404',
            ),
            info=ToolBoxInfoBM(
                exec_code=info.get('exec_code', 404),
                help_text=info.get('help_text', '404')
            ) if info else ToolBoxInfoBM(
                exec_code=404,
                help_text='404'
            ),
            origin=origin
        ).as_result()

    @classmethod
    def stream(cls,
               stream_generator: Any,  # Renamed from source for clarity
               content_type: str = "text/event-stream",  # Default to SSE
               headers: dict | None = None,
               info: str = "OK",
               interface: ToolBoxInterfaces = ToolBoxInterfaces.remote,
               cleanup_func: Callable[[], None] | Callable[[], T] | Callable[[], AsyncGenerator[T, None]] | None = None):
        """
        Create a streaming response Result. Handles SSE and other stream types.

        Args:
            stream_generator: Any stream source (async generator, sync generator, iterable, or single item).
            content_type: Content-Type header (default: text/event-stream for SSE).
            headers: Additional HTTP headers for the response.
            info: Help text for the result.
            interface: Interface to send data to.
            cleanup_func: Optional function for cleanup.

        Returns:
            A Result object configured for streaming.
        """
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=0, help_text=info)

        final_generator: AsyncGenerator[str, None]

        if content_type == "text/event-stream":
            # For SSE, always use SSEGenerator.create_sse_stream to wrap the source.
            # SSEGenerator.create_sse_stream handles various types of stream_generator internally.
            final_generator = SSEGenerator.create_sse_stream(source=stream_generator, cleanup_func=cleanup_func)

            # Standard SSE headers for the HTTP response itself
            # These will be stored in the Result object. Rust side decides how to use them.
            standard_sse_headers = {
                "Cache-Control": "no-cache",  # SSE specific
                "Connection": "keep-alive",  # SSE specific
                "X-Accel-Buffering": "no",  # Useful for proxies with SSE
                # Content-Type is implicitly text/event-stream, will be in streaming_data below
            }
            all_response_headers = standard_sse_headers.copy()
            if headers:
                all_response_headers.update(headers)
        else:
            # For non-SSE streams.
            # If stream_generator is sync, wrap it to be async.
            # If already async or single item, it will be handled.
            # Rust's stream_generator in ToolboxClient seems to handle both sync/async Python generators.
            # For consistency with how SSEGenerator does it, we can wrap sync ones.
            if inspect.isgenerator(stream_generator) or \
                (not isinstance(stream_generator, str) and hasattr(stream_generator, '__iter__')):
                final_generator = SSEGenerator.wrap_sync_generator(stream_generator)  # Simple async wrapper
            elif inspect.isasyncgen(stream_generator):
                final_generator = stream_generator
            else:  # Single item or string
                async def _single_item_gen():
                    yield stream_generator

                final_generator = _single_item_gen()
            all_response_headers = headers if headers else {}

        # Prepare streaming data to be stored in the Result object
        streaming_data = {
            "type": "stream",  # Indicator for Rust side
            "generator": final_generator,
            "content_type": content_type,  # Let Rust know the intended content type
            "headers": all_response_headers  # Intended HTTP headers for the overall response
        }

        result_payload = ToolBoxResult(
            data_to=interface,
            data=streaming_data,
            data_info="Streaming response" if content_type != "text/event-stream" else "SSE Event Stream",
            data_type="stream"  # Generic type for Rust to identify it needs to stream from 'generator'
        )

        return cls(error=error, info=info_obj, result=result_payload)

    @classmethod
    def sse(cls,
            stream_generator: Any,
            info: str = "OK",
            interface: ToolBoxInterfaces = ToolBoxInterfaces.remote,
            cleanup_func: Callable[[], None] | Callable[[], T] | Callable[[], AsyncGenerator[T, None]] | None = None,
            # http_headers: Optional[dict] = None # If we want to allow overriding default SSE HTTP headers
            ):
        """
        Create an Server-Sent Events (SSE) streaming response Result.

        Args:
            stream_generator: A source yielding individual data items. This can be an
                              async generator, sync generator, iterable, or a single item.
                              Each item will be formatted as an SSE event.
            info: Optional help text for the Result.
            interface: Optional ToolBoxInterface to target.
            cleanup_func: Optional cleanup function to run when the stream ends or is cancelled.
            #http_headers: Optional dictionary of custom HTTP headers for the SSE response.

        Returns:
            A Result object configured for SSE streaming.
        """
        # Result.stream will handle calling SSEGenerator.create_sse_stream
        # and setting appropriate default headers for SSE when content_type is "text/event-stream".
        return cls.stream(
            stream_generator=stream_generator,
            content_type="text/event-stream",
            # headers=http_headers, # Pass if we add http_headers param
            info=info,
            interface=interface,
            cleanup_func=cleanup_func
        )

    @classmethod
    def default(cls, interface=ToolBoxInterfaces.native):
        error = ToolBoxError.none
        info = ToolBoxInfo(exec_code=-1, help_text="")
        result = ToolBoxResult(data_to=interface)
        return cls(error=error, info=info, result=result)

    @classmethod
    def json(cls, data, info="OK", interface=ToolBoxInterfaces.remote, exec_code=0, status_code=None):
        """Create a JSON response Result."""
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=status_code or exec_code, help_text=info)

        result = ToolBoxResult(
            data_to=interface,
            data=data,
            data_info="JSON response",
            data_type="json"
        )

        return cls(error=error, info=info_obj, result=result)

    @classmethod
    def text(cls, text_data, content_type="text/plain",exec_code=None,status=200, info="OK", interface=ToolBoxInterfaces.remote, headers=None):
        """Create a text response Result with specific content type."""
        if headers is not None:
            return cls.html(text_data, status= exec_code or status, info=info, headers=headers)
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=exec_code or status, help_text=info)

        result = ToolBoxResult(
            data_to=interface,
            data=text_data,
            data_info="Text response",
            data_type=content_type
        )

        return cls(error=error, info=info_obj, result=result)

    @classmethod
    def binary(cls, data, content_type="application/octet-stream", download_name=None, info="OK",
               interface=ToolBoxInterfaces.remote):
        """Create a binary data response Result."""
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=0, help_text=info)

        # Create a dictionary with binary data and metadata
        binary_data = {
            "data": data,
            "content_type": content_type,
            "filename": download_name
        }

        result = ToolBoxResult(
            data_to=interface,
            data=binary_data,
            data_info=f"Binary response: {download_name}" if download_name else "Binary response",
            data_type="binary"
        )

        return cls(error=error, info=info_obj, result=result)

    @classmethod
    def file(cls, data, filename, content_type=None, info="OK", interface=ToolBoxInterfaces.remote):
        """Create a file download response Result.

        Args:
            data: File data as bytes or base64 string
            filename: Name of the file for download
            content_type: MIME type of the file (auto-detected if None)
            info: Response info text
            interface: Target interface

        Returns:
            Result object configured for file download
        """
        import base64
        import mimetypes

        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=200, help_text=info)

        # Auto-detect content type if not provided
        if content_type is None:
            content_type, _ = mimetypes.guess_type(filename)
            if content_type is None:
                content_type = "application/octet-stream"

        # Ensure data is base64 encoded string (as expected by Rust server)
        if isinstance(data, bytes):
            base64_data = base64.b64encode(data).decode('utf-8')
        elif isinstance(data, str):
            # Assume it's already base64 encoded
            base64_data = data
        else:
            raise ValueError("File data must be bytes or base64 string")

        result = ToolBoxResult(
            data_to=interface,
            data=base64_data,  # Rust expects base64 string for "file" type
            data_info=f"File download: {filename}",
            data_type="file"
        )

        return cls(error=error, info=info_obj, result=result)

    @classmethod
    def redirect(cls, url, status_code=302, info="Redirect", interface=ToolBoxInterfaces.remote):
        """Create a redirect response."""
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=status_code, help_text=info)

        result = ToolBoxResult(
            data_to=interface,
            data=url,
            data_info="Redirect response",
            data_type="redirect"
        )

        return cls(error=error, info=info_obj, result=result)

    @classmethod
    def ok(cls, data=None, data_info="", info="OK", interface=ToolBoxInterfaces.native):
        error = ToolBoxError.none
        info = ToolBoxInfo(exec_code=0, help_text=info)
        result = ToolBoxResult(data_to=interface, data=data, data_info=data_info, data_type=type(data).__name__)
        return cls(error=error, info=info, result=result)

    @classmethod
    def html(cls, data=None, data_info="", info="OK", interface=ToolBoxInterfaces.remote, data_type="html",status=200, headers=None, row=False):
        error = ToolBoxError.none
        info = ToolBoxInfo(exec_code=status, help_text=info)
        from ...utils.system.getting_and_closing_app import get_app

        if not row and not '"<div class="main-content""' in data:
            data = f'<div class="main-content frosted-glass">{data}<div>'
        if not row and not get_app().web_context() in data:
            data = get_app().web_context() + data

        if isinstance(headers, dict):
            result = ToolBoxResult(data_to=interface, data={'html':data,'headers':headers}, data_info=data_info,
                                   data_type="special_html")
        else:
            result = ToolBoxResult(data_to=interface, data=data, data_info=data_info,
                                   data_type=data_type if data_type is not None else type(data).__name__)
        return cls(error=error, info=info, result=result)

    @classmethod
    def future(cls, data=None, data_info="", info="OK", interface=ToolBoxInterfaces.future):
        error = ToolBoxError.none
        info = ToolBoxInfo(exec_code=0, help_text=info)
        result = ToolBoxResult(data_to=interface, data=data, data_info=data_info, data_type="future")
        return cls(error=error, info=info, result=result)

    @classmethod
    def custom_error(cls, data=None, data_info="", info="", exec_code=-1, interface=ToolBoxInterfaces.native):
        error = ToolBoxError.custom_error
        info = ToolBoxInfo(exec_code=exec_code, help_text=info)
        result = ToolBoxResult(data_to=interface, data=data, data_info=data_info, data_type=type(data).__name__)
        return cls(error=error, info=info, result=result)

    @classmethod
    def error(cls, data=None, data_info="", info="", exec_code=450, interface=ToolBoxInterfaces.remote):
        error = ToolBoxError.custom_error
        info = ToolBoxInfo(exec_code=exec_code, help_text=info)
        result = ToolBoxResult(data_to=interface, data=data, data_info=data_info, data_type=type(data).__name__)
        return cls(error=error, info=info, result=result)

    @classmethod
    def default_user_error(cls, info="", exec_code=-3, interface=ToolBoxInterfaces.native, data=None):
        error = ToolBoxError.input_error
        info = ToolBoxInfo(exec_code, info)
        result = ToolBoxResult(data_to=interface, data=data, data_type=type(data).__name__)
        return cls(error=error, info=info, result=result)

    @classmethod
    def default_internal_error(cls, info="", exec_code=-2, interface=ToolBoxInterfaces.native, data=None):
        error = ToolBoxError.internal_error
        info = ToolBoxInfo(exec_code, info)
        result = ToolBoxResult(data_to=interface, data=data, data_type=type(data).__name__)
        return cls(error=error, info=info, result=result)

    def print(self, show=True, show_data=True, prifix="", full_data=False):
        data = '\n' + f"{((prifix + f'Data_{self.result.data_type}: ' + str(self.result.data) if self.result.data is not None else 'NO Data') if not isinstance(self.result.data, Result) else self.result.data.print(show=False, show_data=show_data, prifix=prifix + '-')) if show_data else 'Data: private'}"
        origin = '\n' + f"{prifix + 'Origin: ' + str(self.origin) if self.origin is not None else 'NO Origin'}"
        text = (f"Function Exec code: {self.info.exec_code}"
                f"\n{prifix}Info's:"
                f" {self.info.help_text} {'<|> ' + str(self.result.data_info) if self.result.data_info is not None else ''}"
                f"{origin}{((data[:100]+'...') if not full_data else (data)) if not data.endswith('NO Data') else ''}\n")
        if not show:
            return text
        print("\n======== Result ========\n" + text + "------- EndOfD -------")
        return self

    def log(self, show_data=True, prifix=""):
        from toolboxv2 import get_logger
        get_logger().debug(self.print(show=False, show_data=show_data, prifix=prifix).replace("\n", " - "))
        return self

    def __str__(self):
        return self.print(show=False, show_data=True)

    def get(self, key=None, default=None):
        data = self.result.data
        if isinstance(data, Result):
            return data.get(key=key, default=default)
        if key is not None and isinstance(data, dict):
            return data.get(key, default)
        return data if data is not None else default

    async def aget(self, key=None, default=None):
        if asyncio.isfuture(self.result.data) or asyncio.iscoroutine(self.result.data) or (
            isinstance(self.result.data_to, Enum) and self.result.data_to.name == ToolBoxInterfaces.future.name):
            data = await self.result.data
        else:
            data = self.get(key=None, default=None)
        if isinstance(data, Result):
            return data.get(key=key, default=default)
        if key is not None and isinstance(data, dict):
            return data.get(key, default)
        return data if data is not None else default

    def lazy_return(self, _=0, data=None, **kwargs):
        flags = ['raise', 'logg', 'user', 'intern']
        flag = flags[_] if isinstance(_, int) else _
        if self.info.exec_code == 0:
            return self if data is None else data if _test_is_result(data) else self.ok(data=data, **kwargs)
        if flag == 'raise':
            raise ValueError(self.print(show=False))
        if flag == 'logg':
            from .. import get_logger
            get_logger().error(self.print(show=False))

        if flag == 'user':
            return self if data is None else data if _test_is_result(data) else self.default_user_error(data=data,
                                                                                                        **kwargs)
        if flag == 'intern':
            return self if data is None else data if _test_is_result(data) else self.default_internal_error(data=data,
                                                                                                            **kwargs)

        return self if data is None else data if _test_is_result(data) else self.custom_error(data=data, **kwargs)

    @property
    def bg_task(self):
        return self._task

__class_getitem__(item)

Enable Result[Type] syntax

Source code in toolboxv2/utils/system/types.py

def __class_getitem__(cls, item):
    """Enable Result[Type] syntax"""

    class TypedResult(cls):
        _generic_type = item

        def __init__(self, *args, **kwargs):
            super().__init__(*args, **kwargs)
            self._generic_type = item

    return TypedResult

binary(data, content_type='application/octet-stream', download_name=None, info='OK', interface=ToolBoxInterfaces.remote) classmethod

Create a binary data response Result.

Source code in toolboxv2/utils/system/types.py

@classmethod
def binary(cls, data, content_type="application/octet-stream", download_name=None, info="OK",
           interface=ToolBoxInterfaces.remote):
    """Create a binary data response Result."""
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=0, help_text=info)

    # Create a dictionary with binary data and metadata
    binary_data = {
        "data": data,
        "content_type": content_type,
        "filename": download_name
    }

    result = ToolBoxResult(
        data_to=interface,
        data=binary_data,
        data_info=f"Binary response: {download_name}" if download_name else "Binary response",
        data_type="binary"
    )

    return cls(error=error, info=info_obj, result=result)

cast_to(target_type)

Cast result to different type

Source code in toolboxv2/utils/system/types.py

def cast_to(self, target_type: Type[T]) -> 'Result[T]':
    """Cast result to different type"""
    new_result = Result(
        error=self.error,
        result=self.result,
        info=self.info,
        origin=self.origin,
        generic_type=target_type
    )
    new_result._generic_type = target_type
    return new_result

file(data, filename, content_type=None, info='OK', interface=ToolBoxInterfaces.remote) classmethod

Create a file download response Result.

Parameters:

Name	Type	Description	Default
data		File data as bytes or base64 string	required
filename		Name of the file for download	required
content_type		MIME type of the file (auto-detected if None)	None
info		Response info text	'OK'
interface		Target interface	remote

Returns:

Type	Description
	Result object configured for file download

Source code in toolboxv2/utils/system/types.py

@classmethod
def file(cls, data, filename, content_type=None, info="OK", interface=ToolBoxInterfaces.remote):
    """Create a file download response Result.

    Args:
        data: File data as bytes or base64 string
        filename: Name of the file for download
        content_type: MIME type of the file (auto-detected if None)
        info: Response info text
        interface: Target interface

    Returns:
        Result object configured for file download
    """
    import base64
    import mimetypes

    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=200, help_text=info)

    # Auto-detect content type if not provided
    if content_type is None:
        content_type, _ = mimetypes.guess_type(filename)
        if content_type is None:
            content_type = "application/octet-stream"

    # Ensure data is base64 encoded string (as expected by Rust server)
    if isinstance(data, bytes):
        base64_data = base64.b64encode(data).decode('utf-8')
    elif isinstance(data, str):
        # Assume it's already base64 encoded
        base64_data = data
    else:
        raise ValueError("File data must be bytes or base64 string")

    result = ToolBoxResult(
        data_to=interface,
        data=base64_data,  # Rust expects base64 string for "file" type
        data_info=f"File download: {filename}",
        data_type="file"
    )

    return cls(error=error, info=info_obj, result=result)

get_type_info()

Get the generic type information

Source code in toolboxv2/utils/system/types.py

def get_type_info(self) -> Optional[Type]:
    """Get the generic type information"""
    return self._generic_type

is_typed()

Check if result has type information

Source code in toolboxv2/utils/system/types.py

def is_typed(self) -> bool:
    """Check if result has type information"""
    return self._generic_type is not None

json(data, info='OK', interface=ToolBoxInterfaces.remote, exec_code=0, status_code=None) classmethod

Create a JSON response Result.

Source code in toolboxv2/utils/system/types.py

@classmethod
def json(cls, data, info="OK", interface=ToolBoxInterfaces.remote, exec_code=0, status_code=None):
    """Create a JSON response Result."""
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=status_code or exec_code, help_text=info)

    result = ToolBoxResult(
        data_to=interface,
        data=data,
        data_info="JSON response",
        data_type="json"
    )

    return cls(error=error, info=info_obj, result=result)

redirect(url, status_code=302, info='Redirect', interface=ToolBoxInterfaces.remote) classmethod

Create a redirect response.

Source code in toolboxv2/utils/system/types.py

@classmethod
def redirect(cls, url, status_code=302, info="Redirect", interface=ToolBoxInterfaces.remote):
    """Create a redirect response."""
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=status_code, help_text=info)

    result = ToolBoxResult(
        data_to=interface,
        data=url,
        data_info="Redirect response",
        data_type="redirect"
    )

    return cls(error=error, info=info_obj, result=result)

sse(stream_generator, info='OK', interface=ToolBoxInterfaces.remote, cleanup_func=None) classmethod

Create an Server-Sent Events (SSE) streaming response Result.

Parameters:

Name	Type	Description	Default
stream_generator	Any	A source yielding individual data items. This can be an async generator, sync generator, iterable, or a single item. Each item will be formatted as an SSE event.	required
info	str	Optional help text for the Result.	'OK'
interface	ToolBoxInterfaces	Optional ToolBoxInterface to target.	remote
cleanup_func	Callable[[], None] | Callable[[], T] | Callable[[], AsyncGenerator[T, None]] | None	Optional cleanup function to run when the stream ends or is cancelled.	None
#http_headers		Optional dictionary of custom HTTP headers for the SSE response.	required

Returns:

Type	Description
	A Result object configured for SSE streaming.

Source code in toolboxv2/utils/system/types.py

@classmethod
def sse(cls,
        stream_generator: Any,
        info: str = "OK",
        interface: ToolBoxInterfaces = ToolBoxInterfaces.remote,
        cleanup_func: Callable[[], None] | Callable[[], T] | Callable[[], AsyncGenerator[T, None]] | None = None,
        # http_headers: Optional[dict] = None # If we want to allow overriding default SSE HTTP headers
        ):
    """
    Create an Server-Sent Events (SSE) streaming response Result.

    Args:
        stream_generator: A source yielding individual data items. This can be an
                          async generator, sync generator, iterable, or a single item.
                          Each item will be formatted as an SSE event.
        info: Optional help text for the Result.
        interface: Optional ToolBoxInterface to target.
        cleanup_func: Optional cleanup function to run when the stream ends or is cancelled.
        #http_headers: Optional dictionary of custom HTTP headers for the SSE response.

    Returns:
        A Result object configured for SSE streaming.
    """
    # Result.stream will handle calling SSEGenerator.create_sse_stream
    # and setting appropriate default headers for SSE when content_type is "text/event-stream".
    return cls.stream(
        stream_generator=stream_generator,
        content_type="text/event-stream",
        # headers=http_headers, # Pass if we add http_headers param
        info=info,
        interface=interface,
        cleanup_func=cleanup_func
    )

stream(stream_generator, content_type='text/event-stream', headers=None, info='OK', interface=ToolBoxInterfaces.remote, cleanup_func=None) classmethod

Create a streaming response Result. Handles SSE and other stream types.

Parameters:

Name	Type	Description	Default
stream_generator	Any	Any stream source (async generator, sync generator, iterable, or single item).	required
content_type	str	Content-Type header (default: text/event-stream for SSE).	'text/event-stream'
headers	dict | None	Additional HTTP headers for the response.	None
info	str	Help text for the result.	'OK'
interface	ToolBoxInterfaces	Interface to send data to.	remote
cleanup_func	Callable[[], None] | Callable[[], T] | Callable[[], AsyncGenerator[T, None]] | None	Optional function for cleanup.	None

Returns:

Type	Description
	A Result object configured for streaming.

Source code in toolboxv2/utils/system/types.py

@classmethod
def stream(cls,
           stream_generator: Any,  # Renamed from source for clarity
           content_type: str = "text/event-stream",  # Default to SSE
           headers: dict | None = None,
           info: str = "OK",
           interface: ToolBoxInterfaces = ToolBoxInterfaces.remote,
           cleanup_func: Callable[[], None] | Callable[[], T] | Callable[[], AsyncGenerator[T, None]] | None = None):
    """
    Create a streaming response Result. Handles SSE and other stream types.

    Args:
        stream_generator: Any stream source (async generator, sync generator, iterable, or single item).
        content_type: Content-Type header (default: text/event-stream for SSE).
        headers: Additional HTTP headers for the response.
        info: Help text for the result.
        interface: Interface to send data to.
        cleanup_func: Optional function for cleanup.

    Returns:
        A Result object configured for streaming.
    """
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=0, help_text=info)

    final_generator: AsyncGenerator[str, None]

    if content_type == "text/event-stream":
        # For SSE, always use SSEGenerator.create_sse_stream to wrap the source.
        # SSEGenerator.create_sse_stream handles various types of stream_generator internally.
        final_generator = SSEGenerator.create_sse_stream(source=stream_generator, cleanup_func=cleanup_func)

        # Standard SSE headers for the HTTP response itself
        # These will be stored in the Result object. Rust side decides how to use them.
        standard_sse_headers = {
            "Cache-Control": "no-cache",  # SSE specific
            "Connection": "keep-alive",  # SSE specific
            "X-Accel-Buffering": "no",  # Useful for proxies with SSE
            # Content-Type is implicitly text/event-stream, will be in streaming_data below
        }
        all_response_headers = standard_sse_headers.copy()
        if headers:
            all_response_headers.update(headers)
    else:
        # For non-SSE streams.
        # If stream_generator is sync, wrap it to be async.
        # If already async or single item, it will be handled.
        # Rust's stream_generator in ToolboxClient seems to handle both sync/async Python generators.
        # For consistency with how SSEGenerator does it, we can wrap sync ones.
        if inspect.isgenerator(stream_generator) or \
            (not isinstance(stream_generator, str) and hasattr(stream_generator, '__iter__')):
            final_generator = SSEGenerator.wrap_sync_generator(stream_generator)  # Simple async wrapper
        elif inspect.isasyncgen(stream_generator):
            final_generator = stream_generator
        else:  # Single item or string
            async def _single_item_gen():
                yield stream_generator

            final_generator = _single_item_gen()
        all_response_headers = headers if headers else {}

    # Prepare streaming data to be stored in the Result object
    streaming_data = {
        "type": "stream",  # Indicator for Rust side
        "generator": final_generator,
        "content_type": content_type,  # Let Rust know the intended content type
        "headers": all_response_headers  # Intended HTTP headers for the overall response
    }

    result_payload = ToolBoxResult(
        data_to=interface,
        data=streaming_data,
        data_info="Streaming response" if content_type != "text/event-stream" else "SSE Event Stream",
        data_type="stream"  # Generic type for Rust to identify it needs to stream from 'generator'
    )

    return cls(error=error, info=info_obj, result=result_payload)

text(text_data, content_type='text/plain', exec_code=None, status=200, info='OK', interface=ToolBoxInterfaces.remote, headers=None) classmethod

Create a text response Result with specific content type.

Source code in toolboxv2/utils/system/types.py

@classmethod
def text(cls, text_data, content_type="text/plain",exec_code=None,status=200, info="OK", interface=ToolBoxInterfaces.remote, headers=None):
    """Create a text response Result with specific content type."""
    if headers is not None:
        return cls.html(text_data, status= exec_code or status, info=info, headers=headers)
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=exec_code or status, help_text=info)

    result = ToolBoxResult(
        data_to=interface,
        data=text_data,
        data_info="Text response",
        data_type=content_type
    )

    return cls(error=error, info=info_obj, result=result)

typed_aget(key=None, default=None) async

Async get data with type validation

Source code in toolboxv2/utils/system/types.py

async def typed_aget(self, key=None, default=None) -> T:
    """Async get data with type validation"""
    data = await self.aget(key, default)

    if self._generic_type and data is not None:
        if not self._validate_type(data, self._generic_type):
            from toolboxv2 import get_logger
            get_logger().warning(f"Type mismatch: expected {self._generic_type}, got {type(data)}")

    return data

typed_get(key=None, default=None)

Get data with type validation

Source code in toolboxv2/utils/system/types.py

def typed_get(self, key=None, default=None) -> T:
    """Get data with type validation"""
    data = self.get(key, default)

    if self._generic_type and data is not None:
        # Validate type matches generic parameter
        if not self._validate_type(data, self._generic_type):
            from toolboxv2 import get_logger
            get_logger().warning(f"Type mismatch: expected {self._generic_type}, got {type(data)}")

    return data

typed_json(data, info='OK', interface=ToolBoxInterfaces.remote, exec_code=0, status_code=None) classmethod

Create JSON result with type information

Source code in toolboxv2/utils/system/types.py

@classmethod
def typed_json(cls, data: T, info="OK", interface=ToolBoxInterfaces.remote, exec_code=0,
               status_code=None) -> 'Result[T]':
    """Create JSON result with type information"""
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=status_code or exec_code, help_text=info)

    result = ToolBoxResult(
        data_to=interface,
        data=data,
        data_info="JSON response",
        data_type="json"
    )

    instance = cls(error=error, info=info_obj, result=result)
    if hasattr(cls, '_generic_type'):
        instance._generic_type = cls._generic_type

    return instance

typed_ok(data, data_info='', info='OK', interface=ToolBoxInterfaces.native) classmethod

Create OK result with type information

Source code in toolboxv2/utils/system/types.py

@classmethod
def typed_ok(cls, data: T, data_info="", info="OK", interface=ToolBoxInterfaces.native) -> 'Result[T]':
    """Create OK result with type information"""
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=0, help_text=info)
    result = ToolBoxResult(data_to=interface, data=data, data_info=data_info, data_type=type(data).__name__)

    instance = cls(error=error, info=info_obj, result=result)
    if hasattr(cls, '_generic_type'):
        instance._generic_type = cls._generic_type

    return instance

toolboxv2.ApiResult

Bases: BaseModel

Source code in toolboxv2/utils/system/types.py

class ApiResult(BaseModel):
    error: None | str= None
    origin: Any | None
    result: ToolBoxResultBM | None = None
    info: ToolBoxInfoBM | None

    def as_result(self):
        return Result(
            error=self.error.value if isinstance(self.error, Enum) else self.error,
            result=ToolBoxResult(
                data_to=self.result.data_to.value if isinstance(self.result.data_to, Enum) else self.result.data_to,
                data_info=self.result.data_info,
                data=self.result.data,
                data_type=self.result.data_type
            ) if self.result else None,
            info=ToolBoxInfo(
                exec_code=self.info.exec_code,
                help_text=self.info.help_text
            ) if self.info else None,
            origin=self.origin
        )

    def to_api_result(self):
        return self

    def print(self, *args, **kwargs):
        res = self.as_result().print(*args, **kwargs)
        if not isinstance(res, str):
            res = res.to_api_result()
        return res

toolboxv2.RequestData

Main class representing the complete request data structure.

Source code in toolboxv2/utils/system/types.py

@dataclass
class RequestData:
    """Main class representing the complete request data structure."""
    request: Request
    session: Session
    session_id: str

    @classmethod
    def from_dict(cls, data: dict[str, Any]) -> 'RequestData':
        """Create a RequestData instance from a dictionary."""
        return cls(
            request=Request.from_dict(data.get('request', {})),
            session=Session.from_dict(data.get('session', {})),
            session_id=data.get('session_id', '')
        )

    def to_dict(self) -> dict[str, Any]:
        """Convert the RequestData object back to a dictionary."""
        return {
            'request': self.request.to_dict(),
            'session': self.session.to_dict(),
            'session_id': self.session_id
        }

    def __getattr__(self, name: str) -> Any:
        """Delegate unknown attributes to the `request` object."""
        # Nur wenn das Attribut nicht direkt in RequestData existiert
        # und auch nicht `session` oder `session_id` ist
        if hasattr(self.request, name):
            return getattr(self.request, name)
        raise AttributeError(f"'RequestData' object has no attribute '{name}'")

    @classmethod
    def moc(cls):
        return cls(
            request=Request.from_dict({
                'content_type': 'application/x-www-form-urlencoded',
                'headers': {
                    'accept': '*/*',
                    'accept-encoding': 'gzip, deflate, br, zstd',
                    'accept-language': 'de-DE,de;q=0.9,en-US;q=0.8,en;q=0.7',
                    'connection': 'keep-alive',
                    'content-length': '107',
                    'content-type': 'application/x-www-form-urlencoded',
                    'cookie': 'session=abc123',
                    'host': 'localhost:8080',
                    'hx-current-url': 'http://localhost:8080/api/TruthSeeker/get_main_ui',
                    'hx-request': 'true',
                    'hx-target': 'estimates-guest_1fc2c9',
                    'hx-trigger': 'config-form-guest_1fc2c9',
                    'origin': 'http://localhost:8080',
                    'referer': 'http://localhost:8080/api/TruthSeeker/get_main_ui',
                    'sec-ch-ua': '"Chromium";v="134", "Not:A-Brand";v="24", "Google Chrome";v="134"',
                    'sec-ch-ua-mobile': '?0',
                    'sec-ch-ua-platform': '"Windows"',
                    'sec-fetch-dest': 'empty',
                    'sec-fetch-mode': 'cors',
                    'sec-fetch-site': 'same-origin',
                    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
                },
                'method': 'POST',
                'path': '/api/TruthSeeker/update_estimates',
                'query_params': {},
                'form_data': {
                    'param1': 'value1',
                    'param2': 'value2'
                }
            }),
            session=Session.from_dict({
                'SiID': '29a2e258e18252e2afd5ff943523f09c82f1bb9adfe382a6f33fc6a8381de898',
                'level': '1',
                'spec': '74eed1c8de06886842e235486c3c2fd6bcd60586998ac5beb87f13c0d1750e1d',
                'user_name': 'root',
                'custom_field': 'custom_value'
            }),
            session_id='0x29dd1ac0d1e30d3f'
        )

__getattr__(name)

Delegate unknown attributes to the request object.

Source code in toolboxv2/utils/system/types.py

def __getattr__(self, name: str) -> Any:
    """Delegate unknown attributes to the `request` object."""
    # Nur wenn das Attribut nicht direkt in RequestData existiert
    # und auch nicht `session` oder `session_id` ist
    if hasattr(self.request, name):
        return getattr(self.request, name)
    raise AttributeError(f"'RequestData' object has no attribute '{name}'")

from_dict(data) classmethod

Create a RequestData instance from a dictionary.

Source code in toolboxv2/utils/system/types.py

@classmethod
def from_dict(cls, data: dict[str, Any]) -> 'RequestData':
    """Create a RequestData instance from a dictionary."""
    return cls(
        request=Request.from_dict(data.get('request', {})),
        session=Session.from_dict(data.get('session', {})),
        session_id=data.get('session_id', '')
    )

to_dict()

Convert the RequestData object back to a dictionary.

Source code in toolboxv2/utils/system/types.py

def to_dict(self) -> dict[str, Any]:
    """Convert the RequestData object back to a dictionary."""
    return {
        'request': self.request.to_dict(),
        'session': self.session.to_dict(),
        'session_id': self.session_id
    }

Security

toolboxv2.Code

Source code in toolboxv2/utils/security/cryp.py

class Code:

    @staticmethod
    def DK():
        return DEVICE_KEY

    @staticmethod
    def generate_random_string(length: int) -> str:
        """
        Generiert eine zufällige Zeichenkette der angegebenen Länge.

        Args:
            length (int): Die Länge der zu generierenden Zeichenkette.

        Returns:
            str: Die generierte Zeichenkette.
        """
        return secrets.token_urlsafe(length)

    def decode_code(self, encrypted_data, key=None):

        if not isinstance(encrypted_data, str):
            encrypted_data = str(encrypted_data)

        if key is None:
            key = DEVICE_KEY()

        return self.decrypt_symmetric(encrypted_data, key)

    def encode_code(self, data, key=None):

        if not isinstance(data, str):
            data = str(data)

        if key is None:
            key = DEVICE_KEY()

        return self.encrypt_symmetric(data, key)

    @staticmethod
    def generate_seed() -> int:
        """
        Erzeugt eine zufällige Zahl als Seed.

        Returns:
            int: Eine zufällige Zahl.
        """
        return random.randint(2 ** 32 - 1, 2 ** 64 - 1)

    @staticmethod
    def one_way_hash(text: str, salt: str = '', pepper: str = '') -> str:
        """
        Erzeugt einen Hash eines gegebenen Textes mit Salt, Pepper und optional einem Seed.

        Args:
            text (str): Der zu hashende Text.
            salt (str): Der Salt-Wert.
            pepper (str): Der Pepper-Wert.
            seed (int, optional): Ein optionaler Seed-Wert. Standardmäßig None.

        Returns:
            str: Der resultierende Hash-Wert.
        """
        return hashlib.sha256((salt + text + pepper).encode()).hexdigest()

    @staticmethod
    def generate_symmetric_key(as_str=True) -> str or bytes:
        """
        Generiert einen Schlüssel für die symmetrische Verschlüsselung.

        Returns:
            str: Der generierte Schlüssel.
        """
        key = Fernet.generate_key()
        if as_str:
            key = key.decode()
        return key

    @staticmethod
    def encrypt_symmetric(text: str or bytes, key: str) -> str:
        """
        Verschlüsselt einen Text mit einem gegebenen symmetrischen Schlüssel.

        Args:
            text (str): Der zu verschlüsselnde Text.
            key (str): Der symmetrische Schlüssel.

        Returns:
            str: Der verschlüsselte Text.
        """
        if isinstance(text, str):
            text = text.encode()

        try:
            fernet = Fernet(key.encode())
            return fernet.encrypt(text).decode()
        except Exception as e:
            get_logger().error(f"Error encrypt_symmetric #{str(e)}#")
            return "Error encrypt"

    @staticmethod
    def decrypt_symmetric(encrypted_text: str, key: str, to_str=True, mute=False) -> str or bytes:
        """
        Entschlüsselt einen Text mit einem gegebenen symmetrischen Schlüssel.

        Args:
            encrypted_text (str): Der zu entschlüsselnde Text.
            key (str): Der symmetrische Schlüssel.
            to_str (bool): default true returns str if false returns bytes
        Returns:
            str: Der entschlüsselte Text.
        """

        if isinstance(key, str):
            key = key.encode()

        #try:
        fernet = Fernet(key)
        text_b = fernet.decrypt(encrypted_text)
        if not to_str:
            return text_b
        return text_b.decode()
        # except Exception as e:
        #     get_logger().error(f"Error decrypt_symmetric {e}")
        #     if not mute:
        #         raise e
        #     if not to_str:
        #         return f"Error decoding".encode()
        #     return f"Error decoding"

    @staticmethod
    def generate_asymmetric_keys() -> (str, str):
        """
        Generiert ein Paar von öffentlichen und privaten Schlüsseln für die asymmetrische Verschlüsselung.

        Args:
            seed (int, optional): Ein optionaler Seed-Wert. Standardmäßig None.

        Returns:
            (str, str): Ein Tupel aus öffentlichem und privatem Schlüssel.
        """
        private_key = rsa.generate_private_key(
            public_exponent=65537,
            key_size=2048 * 3,
        )
        public_key = private_key.public_key()

        # Serialisieren der Schlüssel
        pem_private_key = private_key.private_bytes(
            encoding=serialization.Encoding.PEM,
            format=serialization.PrivateFormat.PKCS8,
            encryption_algorithm=serialization.NoEncryption()
        ).decode()

        pem_public_key = public_key.public_bytes(
            encoding=serialization.Encoding.PEM,
            format=serialization.PublicFormat.SubjectPublicKeyInfo
        ).decode()

        return pem_public_key, pem_private_key

    @staticmethod
    def save_keys_to_files(public_key: str, private_key: str, directory: str = "keys") -> None:
        """
        Speichert die generierten Schlüssel in separate Dateien.
        Der private Schlüssel wird mit dem Device Key verschlüsselt.

        Args:
            public_key (str): Der öffentliche Schlüssel im PEM-Format
            private_key (str): Der private Schlüssel im PEM-Format
            directory (str): Das Verzeichnis, in dem die Schlüssel gespeichert werden sollen
        """
        # Erstelle das Verzeichnis, falls es nicht existiert
        os.makedirs(directory, exist_ok=True)

        # Hole den Device Key
        device_key = DEVICE_KEY()

        # Verschlüssele den privaten Schlüssel mit dem Device Key
        encrypted_private_key = Code.encrypt_symmetric(private_key, device_key)

        # Speichere den öffentlichen Schlüssel
        public_key_path = os.path.join(directory, "public_key.pem")
        with open(public_key_path, "w") as f:
            f.write(public_key)

        # Speichere den verschlüsselten privaten Schlüssel
        private_key_path = os.path.join(directory, "private_key.pem")
        with open(private_key_path, "w") as f:
            f.write(encrypted_private_key)

        print("Saved keys in ", public_key_path)

    @staticmethod
    def load_keys_from_files(directory: str = "keys") -> (str, str):
        """
        Lädt die Schlüssel aus den Dateien.
        Der private Schlüssel wird mit dem Device Key entschlüsselt.

        Args:
            directory (str): Das Verzeichnis, aus dem die Schlüssel geladen werden sollen

        Returns:
            (str, str): Ein Tupel aus öffentlichem und privatem Schlüssel

        Raises:
            FileNotFoundError: Wenn die Schlüsseldateien nicht gefunden werden können
        """
        # Pfade zu den Schlüsseldateien
        public_key_path = os.path.join(directory, "public_key.pem")
        private_key_path = os.path.join(directory, "private_key.pem")

        # Prüfe ob die Dateien existieren
        if not os.path.exists(public_key_path) or not os.path.exists(private_key_path):
            return "", ""

        # Hole den Device Key
        device_key = DEVICE_KEY()

        # Lade den öffentlichen Schlüssel
        with open(public_key_path) as f:
            public_key = f.read()

        # Lade und entschlüssele den privaten Schlüssel
        with open(private_key_path) as f:
            encrypted_private_key = f.read()
            private_key = Code.decrypt_symmetric(encrypted_private_key, device_key)

        return public_key, private_key

    @staticmethod
    def encrypt_asymmetric(text: str, public_key_str: str) -> str:
        """
        Verschlüsselt einen Text mit einem gegebenen öffentlichen Schlüssel.

        Args:
            text (str): Der zu verschlüsselnde Text.
            public_key_str (str): Der öffentliche Schlüssel als String oder im pem format.

        Returns:
            str: Der verschlüsselte Text.
        """
        # try:
        #    public_key: RSAPublicKey = serialization.load_pem_public_key(public_key_str.encode())
        #  except Exception as e:
        #     get_logger().error(f"Error encrypt_asymmetric {e}")
        try:
            public_key: RSAPublicKey = serialization.load_pem_public_key(public_key_str.encode())
            encrypted = public_key.encrypt(
                text.encode(),
                padding.OAEP(
                    mgf=padding.MGF1(algorithm=hashes.SHA512()),
                    algorithm=hashes.SHA512(),
                    label=None
                )
            )
            return encrypted.hex()
        except Exception as e:
            get_logger().error(f"Error encrypt_asymmetric {e}")
            return "Invalid"

    @staticmethod
    def decrypt_asymmetric(encrypted_text_hex: str, private_key_str: str) -> str:
        """
        Entschlüsselt einen Text mit einem gegebenen privaten Schlüssel.

        Args:
            encrypted_text_hex (str): Der verschlüsselte Text als Hex-String.
            private_key_str (str): Der private Schlüssel als String.

        Returns:
            str: Der entschlüsselte Text.
        """
        try:
            private_key = serialization.load_pem_private_key(private_key_str.encode(), password=None)
            decrypted = private_key.decrypt(
                bytes.fromhex(encrypted_text_hex),
                padding.OAEP(
                    mgf=padding.MGF1(algorithm=hashes.SHA512()),
                    algorithm=hashes.SHA512(),
                    label=None
                )
            )
            return decrypted.decode()

        except Exception as e:
            get_logger().error(f"Error decrypt_asymmetric {e}")
        return "Invalid"

    @staticmethod
    def verify_signature(signature: str or bytes, message: str or bytes, public_key_str: str,
                         salt_length=padding.PSS.MAX_LENGTH) -> bool:
        if isinstance(signature, str):
            signature = signature.encode()
        if isinstance(message, str):
            message = message.encode()
        try:
            public_key: RSAPublicKey = serialization.load_pem_public_key(public_key_str.encode())
            public_key.verify(
                signature=signature,
                data=message,
                padding=padding.PSS(
                    mgf=padding.MGF1(hashes.SHA512()),
                    salt_length=salt_length
                ),
                algorithm=hashes.SHA512()
            )
            return True
        except:
            pass
        return False

    @staticmethod
    def verify_signature_web_algo(signature: str or bytes, message: str or bytes, public_key_str: str,
                                  algo: int = -512) -> bool:
        signature_algorithm = ECDSA(hashes.SHA512())
        if algo != -512:
            signature_algorithm = ECDSA(hashes.SHA256())

        if isinstance(signature, str):
            signature = signature.encode()
        if isinstance(message, str):
            message = message.encode()
        try:
            public_key = serialization.load_pem_public_key(public_key_str.encode())
            public_key.verify(
                signature=signature,
                data=message,
                # padding=padding.PSS(
                #    mgf=padding.MGF1(hashes.SHA512()),
                #    salt_length=padding.PSS.MAX_LENGTH
                # ),
                signature_algorithm=signature_algorithm
            )
            return True
        except:
            pass
        return False

    @staticmethod
    def create_signature(message: str, private_key_str: str, salt_length=padding.PSS.MAX_LENGTH,
                         row=False) -> str or bytes:
        try:
            private_key = serialization.load_pem_private_key(private_key_str.encode(), password=None)
            signature = private_key.sign(
                message.encode(),
                padding.PSS(
                    mgf=padding.MGF1(hashes.SHA512()),
                    salt_length=salt_length
                ),
                hashes.SHA512()
            )
            if row:
                return signature
            return base64.b64encode(signature).decode()
        except Exception as e:
            get_logger().error(f"Error create_signature {e}")
            print(e)
        return "Invalid Key"

    @staticmethod
    def pem_to_public_key(pem_key: str):
        """
        Konvertiert einen PEM-kodierten öffentlichen Schlüssel in ein PublicKey-Objekt.

        Args:
            pem_key (str): Der PEM-kodierte öffentliche Schlüssel.

        Returns:
            PublicKey: Das PublicKey-Objekt.
        """
        public_key = serialization.load_pem_public_key(pem_key.encode())
        return public_key

    @staticmethod
    def public_key_to_pem(public_key: RSAPublicKey):
        """
        Konvertiert ein PublicKey-Objekt in einen PEM-kodierten String.

        Args:
            public_key (PublicKey): Das PublicKey-Objekt.

        Returns:
            str: Der PEM-kodierte öffentliche Schlüssel.
        """
        pem = public_key.public_bytes(
            encoding=serialization.Encoding.PEM,
            format=serialization.PublicFormat.SubjectPublicKeyInfo
        )
        return pem.decode()

decrypt_asymmetric(encrypted_text_hex, private_key_str) staticmethod

Entschlüsselt einen Text mit einem gegebenen privaten Schlüssel.

Parameters:

Name	Type	Description	Default
encrypted_text_hex	str	Der verschlüsselte Text als Hex-String.	required
private_key_str	str	Der private Schlüssel als String.	required

Returns:

Name	Type	Description
str	str	Der entschlüsselte Text.

Source code in toolboxv2/utils/security/cryp.py

@staticmethod
def decrypt_asymmetric(encrypted_text_hex: str, private_key_str: str) -> str:
    """
    Entschlüsselt einen Text mit einem gegebenen privaten Schlüssel.

    Args:
        encrypted_text_hex (str): Der verschlüsselte Text als Hex-String.
        private_key_str (str): Der private Schlüssel als String.

    Returns:
        str: Der entschlüsselte Text.
    """
    try:
        private_key = serialization.load_pem_private_key(private_key_str.encode(), password=None)
        decrypted = private_key.decrypt(
            bytes.fromhex(encrypted_text_hex),
            padding.OAEP(
                mgf=padding.MGF1(algorithm=hashes.SHA512()),
                algorithm=hashes.SHA512(),
                label=None
            )
        )
        return decrypted.decode()

    except Exception as e:
        get_logger().error(f"Error decrypt_asymmetric {e}")
    return "Invalid"

decrypt_symmetric(encrypted_text, key, to_str=True, mute=False) staticmethod

Entschlüsselt einen Text mit einem gegebenen symmetrischen Schlüssel.

Parameters:

Name	Type	Description	Default
encrypted_text	str	Der zu entschlüsselnde Text.	required
key	str	Der symmetrische Schlüssel.	required
to_str	bool	default true returns str if false returns bytes	True

Returns: str: Der entschlüsselte Text.

Source code in toolboxv2/utils/security/cryp.py

@staticmethod
def decrypt_symmetric(encrypted_text: str, key: str, to_str=True, mute=False) -> str or bytes:
    """
    Entschlüsselt einen Text mit einem gegebenen symmetrischen Schlüssel.

    Args:
        encrypted_text (str): Der zu entschlüsselnde Text.
        key (str): Der symmetrische Schlüssel.
        to_str (bool): default true returns str if false returns bytes
    Returns:
        str: Der entschlüsselte Text.
    """

    if isinstance(key, str):
        key = key.encode()

    #try:
    fernet = Fernet(key)
    text_b = fernet.decrypt(encrypted_text)
    if not to_str:
        return text_b
    return text_b.decode()

encrypt_asymmetric(text, public_key_str) staticmethod

Verschlüsselt einen Text mit einem gegebenen öffentlichen Schlüssel.

Parameters:

Name	Type	Description	Default
text	str	Der zu verschlüsselnde Text.	required
public_key_str	str	Der öffentliche Schlüssel als String oder im pem format.	required

Returns:

Name	Type	Description
str	str	Der verschlüsselte Text.

Source code in toolboxv2/utils/security/cryp.py

@staticmethod
def encrypt_asymmetric(text: str, public_key_str: str) -> str:
    """
    Verschlüsselt einen Text mit einem gegebenen öffentlichen Schlüssel.

    Args:
        text (str): Der zu verschlüsselnde Text.
        public_key_str (str): Der öffentliche Schlüssel als String oder im pem format.

    Returns:
        str: Der verschlüsselte Text.
    """
    # try:
    #    public_key: RSAPublicKey = serialization.load_pem_public_key(public_key_str.encode())
    #  except Exception as e:
    #     get_logger().error(f"Error encrypt_asymmetric {e}")
    try:
        public_key: RSAPublicKey = serialization.load_pem_public_key(public_key_str.encode())
        encrypted = public_key.encrypt(
            text.encode(),
            padding.OAEP(
                mgf=padding.MGF1(algorithm=hashes.SHA512()),
                algorithm=hashes.SHA512(),
                label=None
            )
        )
        return encrypted.hex()
    except Exception as e:
        get_logger().error(f"Error encrypt_asymmetric {e}")
        return "Invalid"

encrypt_symmetric(text, key) staticmethod

Verschlüsselt einen Text mit einem gegebenen symmetrischen Schlüssel.

Parameters:

Name	Type	Description	Default
text	str	Der zu verschlüsselnde Text.	required
key	str	Der symmetrische Schlüssel.	required

Returns:

Name	Type	Description
str	str	Der verschlüsselte Text.

Source code in toolboxv2/utils/security/cryp.py

@staticmethod
def encrypt_symmetric(text: str or bytes, key: str) -> str:
    """
    Verschlüsselt einen Text mit einem gegebenen symmetrischen Schlüssel.

    Args:
        text (str): Der zu verschlüsselnde Text.
        key (str): Der symmetrische Schlüssel.

    Returns:
        str: Der verschlüsselte Text.
    """
    if isinstance(text, str):
        text = text.encode()

    try:
        fernet = Fernet(key.encode())
        return fernet.encrypt(text).decode()
    except Exception as e:
        get_logger().error(f"Error encrypt_symmetric #{str(e)}#")
        return "Error encrypt"

generate_asymmetric_keys() staticmethod

Generiert ein Paar von öffentlichen und privaten Schlüsseln für die asymmetrische Verschlüsselung.

Parameters:

Name	Type	Description	Default
seed	int	Ein optionaler Seed-Wert. Standardmäßig None.	required

Returns:

Type	Description
(str, str)	Ein Tupel aus öffentlichem und privatem Schlüssel.

Source code in toolboxv2/utils/security/cryp.py

@staticmethod
def generate_asymmetric_keys() -> (str, str):
    """
    Generiert ein Paar von öffentlichen und privaten Schlüsseln für die asymmetrische Verschlüsselung.

    Args:
        seed (int, optional): Ein optionaler Seed-Wert. Standardmäßig None.

    Returns:
        (str, str): Ein Tupel aus öffentlichem und privatem Schlüssel.
    """
    private_key = rsa.generate_private_key(
        public_exponent=65537,
        key_size=2048 * 3,
    )
    public_key = private_key.public_key()

    # Serialisieren der Schlüssel
    pem_private_key = private_key.private_bytes(
        encoding=serialization.Encoding.PEM,
        format=serialization.PrivateFormat.PKCS8,
        encryption_algorithm=serialization.NoEncryption()
    ).decode()

    pem_public_key = public_key.public_bytes(
        encoding=serialization.Encoding.PEM,
        format=serialization.PublicFormat.SubjectPublicKeyInfo
    ).decode()

    return pem_public_key, pem_private_key

generate_random_string(length) staticmethod

Generiert eine zufällige Zeichenkette der angegebenen Länge.

Parameters:

Name	Type	Description	Default
length	int	Die Länge der zu generierenden Zeichenkette.	required

Returns:

Name	Type	Description
str	str	Die generierte Zeichenkette.

Source code in toolboxv2/utils/security/cryp.py

@staticmethod
def generate_random_string(length: int) -> str:
    """
    Generiert eine zufällige Zeichenkette der angegebenen Länge.

    Args:
        length (int): Die Länge der zu generierenden Zeichenkette.

    Returns:
        str: Die generierte Zeichenkette.
    """
    return secrets.token_urlsafe(length)

generate_seed() staticmethod

Erzeugt eine zufällige Zahl als Seed.

Returns:

Name	Type	Description
int	int	Eine zufällige Zahl.

Source code in toolboxv2/utils/security/cryp.py

@staticmethod
def generate_seed() -> int:
    """
    Erzeugt eine zufällige Zahl als Seed.

    Returns:
        int: Eine zufällige Zahl.
    """
    return random.randint(2 ** 32 - 1, 2 ** 64 - 1)

generate_symmetric_key(as_str=True) staticmethod

Generiert einen Schlüssel für die symmetrische Verschlüsselung.

Returns:

Name	Type	Description
str	str or bytes	Der generierte Schlüssel.

Source code in toolboxv2/utils/security/cryp.py

@staticmethod
def generate_symmetric_key(as_str=True) -> str or bytes:
    """
    Generiert einen Schlüssel für die symmetrische Verschlüsselung.

    Returns:
        str: Der generierte Schlüssel.
    """
    key = Fernet.generate_key()
    if as_str:
        key = key.decode()
    return key

load_keys_from_files(directory='keys') staticmethod

Lädt die Schlüssel aus den Dateien. Der private Schlüssel wird mit dem Device Key entschlüsselt.

Parameters:

Name	Type	Description	Default
directory	str	Das Verzeichnis, aus dem die Schlüssel geladen werden sollen	'keys'

Returns:

Type	Description
(str, str)	Ein Tupel aus öffentlichem und privatem Schlüssel

Raises:

Type	Description
FileNotFoundError	Wenn die Schlüsseldateien nicht gefunden werden können

Source code in toolboxv2/utils/security/cryp.py

@staticmethod
def load_keys_from_files(directory: str = "keys") -> (str, str):
    """
    Lädt die Schlüssel aus den Dateien.
    Der private Schlüssel wird mit dem Device Key entschlüsselt.

    Args:
        directory (str): Das Verzeichnis, aus dem die Schlüssel geladen werden sollen

    Returns:
        (str, str): Ein Tupel aus öffentlichem und privatem Schlüssel

    Raises:
        FileNotFoundError: Wenn die Schlüsseldateien nicht gefunden werden können
    """
    # Pfade zu den Schlüsseldateien
    public_key_path = os.path.join(directory, "public_key.pem")
    private_key_path = os.path.join(directory, "private_key.pem")

    # Prüfe ob die Dateien existieren
    if not os.path.exists(public_key_path) or not os.path.exists(private_key_path):
        return "", ""

    # Hole den Device Key
    device_key = DEVICE_KEY()

    # Lade den öffentlichen Schlüssel
    with open(public_key_path) as f:
        public_key = f.read()

    # Lade und entschlüssele den privaten Schlüssel
    with open(private_key_path) as f:
        encrypted_private_key = f.read()
        private_key = Code.decrypt_symmetric(encrypted_private_key, device_key)

    return public_key, private_key

one_way_hash(text, salt='', pepper='') staticmethod

Erzeugt einen Hash eines gegebenen Textes mit Salt, Pepper und optional einem Seed.

Parameters:

Name	Type	Description	Default
text	str	Der zu hashende Text.	required
salt	str	Der Salt-Wert.	''
pepper	str	Der Pepper-Wert.	''
seed	int	Ein optionaler Seed-Wert. Standardmäßig None.	required

Returns:

Name	Type	Description
str	str	Der resultierende Hash-Wert.

Source code in toolboxv2/utils/security/cryp.py

@staticmethod
def one_way_hash(text: str, salt: str = '', pepper: str = '') -> str:
    """
    Erzeugt einen Hash eines gegebenen Textes mit Salt, Pepper und optional einem Seed.

    Args:
        text (str): Der zu hashende Text.
        salt (str): Der Salt-Wert.
        pepper (str): Der Pepper-Wert.
        seed (int, optional): Ein optionaler Seed-Wert. Standardmäßig None.

    Returns:
        str: Der resultierende Hash-Wert.
    """
    return hashlib.sha256((salt + text + pepper).encode()).hexdigest()

pem_to_public_key(pem_key) staticmethod

Konvertiert einen PEM-kodierten öffentlichen Schlüssel in ein PublicKey-Objekt.

Parameters:

Name	Type	Description	Default
pem_key	str	Der PEM-kodierte öffentliche Schlüssel.	required

Returns:

Name	Type	Description
PublicKey		Das PublicKey-Objekt.

Source code in toolboxv2/utils/security/cryp.py

@staticmethod
def pem_to_public_key(pem_key: str):
    """
    Konvertiert einen PEM-kodierten öffentlichen Schlüssel in ein PublicKey-Objekt.

    Args:
        pem_key (str): Der PEM-kodierte öffentliche Schlüssel.

    Returns:
        PublicKey: Das PublicKey-Objekt.
    """
    public_key = serialization.load_pem_public_key(pem_key.encode())
    return public_key

public_key_to_pem(public_key) staticmethod

Konvertiert ein PublicKey-Objekt in einen PEM-kodierten String.

Parameters:

Name	Type	Description	Default
public_key	PublicKey	Das PublicKey-Objekt.	required

Returns:

Name	Type	Description
str		Der PEM-kodierte öffentliche Schlüssel.

Source code in toolboxv2/utils/security/cryp.py

@staticmethod
def public_key_to_pem(public_key: RSAPublicKey):
    """
    Konvertiert ein PublicKey-Objekt in einen PEM-kodierten String.

    Args:
        public_key (PublicKey): Das PublicKey-Objekt.

    Returns:
        str: Der PEM-kodierte öffentliche Schlüssel.
    """
    pem = public_key.public_bytes(
        encoding=serialization.Encoding.PEM,
        format=serialization.PublicFormat.SubjectPublicKeyInfo
    )
    return pem.decode()

save_keys_to_files(public_key, private_key, directory='keys') staticmethod

Speichert die generierten Schlüssel in separate Dateien. Der private Schlüssel wird mit dem Device Key verschlüsselt.

Parameters:

Name	Type	Description	Default
public_key	str	Der öffentliche Schlüssel im PEM-Format	required
private_key	str	Der private Schlüssel im PEM-Format	required
directory	str	Das Verzeichnis, in dem die Schlüssel gespeichert werden sollen	'keys'

Source code in toolboxv2/utils/security/cryp.py

@staticmethod
def save_keys_to_files(public_key: str, private_key: str, directory: str = "keys") -> None:
    """
    Speichert die generierten Schlüssel in separate Dateien.
    Der private Schlüssel wird mit dem Device Key verschlüsselt.

    Args:
        public_key (str): Der öffentliche Schlüssel im PEM-Format
        private_key (str): Der private Schlüssel im PEM-Format
        directory (str): Das Verzeichnis, in dem die Schlüssel gespeichert werden sollen
    """
    # Erstelle das Verzeichnis, falls es nicht existiert
    os.makedirs(directory, exist_ok=True)

    # Hole den Device Key
    device_key = DEVICE_KEY()

    # Verschlüssele den privaten Schlüssel mit dem Device Key
    encrypted_private_key = Code.encrypt_symmetric(private_key, device_key)

    # Speichere den öffentlichen Schlüssel
    public_key_path = os.path.join(directory, "public_key.pem")
    with open(public_key_path, "w") as f:
        f.write(public_key)

    # Speichere den verschlüsselten privaten Schlüssel
    private_key_path = os.path.join(directory, "private_key.pem")
    with open(private_key_path, "w") as f:
        f.write(encrypted_private_key)

    print("Saved keys in ", public_key_path)

Modules & Flows

toolboxv2.mods

Canvas

Tools

Bases: MainTool

Source code in toolboxv2/mods/Canvas.py

class Tools(MainTool):  # Removed EventManager for simplicity, as it was causing the issue. Direct SSE is better here.
    def __init__(self, app: App):
        self.name = MOD_NAME
        self.version = VERSION
        self.color = "GREEN"
        self.tools_dict = {"name": MOD_NAME, "Version": self.show_version}

        # Canvas specific state
        self.live_canvas_sessions: dict[str, list[asyncio.Queue]] = defaultdict(list)
        self.active_user_previews: dict[str, dict[str, Any]] = defaultdict(dict)
        self.previews_lock = asyncio.Lock()

        MainTool.__init__(self, load=on_start, v=self.version, tool=self.tools_dict, name=self.name,
                          color=self.color, app=app)
        self.app.logger.info(f"Canvas Tools (v{self.version}) initialized for app {self.app.id}.")

    @property
    def db_mod(self):
        db = self.app.get_mod("DB", spec=Name)
        if db.mode.value != "CLUSTER_BLOB":
            db.edit_cli("CB")
        return db

    def _broadcast_to_canvas_listeners(self, canvas_id: str, event_type: str, data: dict[str, Any],
                                       originator_user_id: str | None = None):
        """
        Creates a broadcast coroutine and submits it to the app's dedicated
        async manager to be run in the background.
        This is now a non-blocking fire-and-forget operation.
        """

        async def broadcast_coro():
            if canvas_id not in self.live_canvas_sessions:
                return

            message_obj = {
                "event": event_type,
                "data": json.dumps({
                    "canvas_id": canvas_id,
                    "originator_user_id": originator_user_id,
                    **data
                })
            }

            listeners = list(self.live_canvas_sessions.get(canvas_id, []))

            for q in listeners:
                try:
                    # Non-blocking put. If the queue is full, the client is lagging,
                    # and it's better to drop a message than to block the server.
                    q.put_nowait(message_obj)
                except asyncio.QueueFull:
                    self.app.logger.warning(
                        f"SSE queue full for canvas {canvas_id}. Message '{event_type}' dropped for one client.")
                except Exception as e:
                    self.app.logger.error(f"Error putting message on SSE queue: {e}")

        # Use the app's robust background runner to execute immediately and not block the caller.
        self.app.run_bg_task(broadcast_coro)

    def show_version(self):
        self.app.logger.info(f"{self.name} Version: {self.version}")
        return self.version

    async def _get_user_specific_db_key(self, request: RequestData, base_key: str) -> str | None:
        # This logic is correct and can remain as is.

        user = await get_user_from_request(self.app, request)
        if user and user.uid:
            return f"{base_key}_{user.uid}"
        self.print("ok")
        # Fallback for public/guest access if you want to support it
        return f"{base_key}_public"

handle_send_canvas_action(app, request, data) async

Handles incremental, real-time actions from clients (e.g., adding an element). It persists the change to the database and then broadcasts it to all live listeners.

Source code in toolboxv2/mods/Canvas.py

@export(mod_name=MOD_NAME, api=True, version=VERSION, name="send_canvas_action", api_methods=['POST'],
        request_as_kwarg=True)
async def handle_send_canvas_action(app: App, request: RequestData, data: dict[str, Any]):
    """
    Handles incremental, real-time actions from clients (e.g., adding an element).
    It persists the change to the database and then broadcasts it to all live listeners.
    """
    canvas_tool = app.get_mod(MOD_NAME)
    if not canvas_tool or not canvas_tool.db_mod:
        return Result.default_internal_error("Canvas module or DB not loaded.")

    if not data:
        return Result.default_user_error("Request data is missing.", 400)

    canvas_id = data.get("canvas_id")
    action_type = data.get("action_type")
    action_payload = data.get("payload")
    user_id = data.get("user_id")

    if not all([canvas_id, action_type, user_id]) or action_payload is None:
        return Result.default_user_error("Request missing required fields.", 400)

    # --- Flow 1: Ephemeral 'preview' actions that DO NOT get persisted ---
    if action_type in ["preview_update", "preview_clear"]:
        sse_event_type = "user_preview_update" if action_type == "preview_update" else "clear_user_preview"
        sse_data = {"user_id": user_id}

        async with canvas_tool.previews_lock:
            if action_type == "preview_update":
                canvas_tool.active_user_previews[canvas_id][user_id] = action_payload
                sse_data["preview_data"] = action_payload
            elif user_id in canvas_tool.active_user_previews.get(canvas_id, {}):
                del canvas_tool.active_user_previews[canvas_id][user_id]

        # MODIFICATION: Call the non-blocking broadcast method. This returns immediately.
        canvas_tool._broadcast_to_canvas_listeners(
            canvas_id=canvas_id, event_type=sse_event_type,
            data=sse_data, originator_user_id=user_id
        )
        return Result.ok(info=f"'{action_type}' broadcasted.")

    # --- Flow 2: Persistent actions that modify the canvas state ---
    if action_type not in ["element_add", "element_update", "element_remove"]:
        return Result.default_user_error(f"Unknown persistent action_type: {action_type}", 400)

    # Load the full, current session state from the database
    user_db_key_base = await canvas_tool._get_user_specific_db_key(request, SESSION_DATA_PREFIX)
    session_db_key = f"{user_db_key_base}_{canvas_id}"
    try:
        db_result = canvas_tool.db_mod.get(session_db_key)
        if not db_result or db_result.is_error() or not db_result.get():
            return Result.default_user_error("Canvas session not found in database.", 404)

        session_data_str = db_result.get()[0] if isinstance(db_result.get(), list) else db_result.get()
        session_data = IdeaSessionData.model_validate_json(session_data_str)
    except Exception as e:
        app.logger.error(f"DB Load/Parse failed for C:{canvas_id}. Error: {e}", exc_info=True)
        return Result.default_internal_error("Could not load canvas data to apply changes.")

    # Apply the action to the in-memory Pydantic object
    if action_type == "element_add":
        session_data.canvas_elements.append(CanvasElement(**action_payload))
    elif action_type == "element_update":
        element_id = action_payload.get("id")
        for i, el in enumerate(session_data.canvas_elements):
            if el.id == element_id:
                session_data.canvas_elements[i] = el.model_copy(update=action_payload)
                break
    elif action_type == "element_remove":
        ids_to_remove = set(action_payload.get("ids", [action_payload.get("id")]))
        session_data.canvas_elements = [el for el in session_data.canvas_elements if el.id not in ids_to_remove]

    # Save the modified object back to the database
    session_data.last_modified = datetime.now(UTC).timestamp()
    canvas_tool.db_mod.set(session_db_key, session_data.model_dump_json(exclude_none=True))

    # Broadcast the successful, persisted action to all connected clients
    # MODIFICATION: Call the non-blocking broadcast method.
    canvas_tool._broadcast_to_canvas_listeners(
        canvas_id=canvas_id,
        event_type="canvas_elements_changed",
        data={"action": action_type, "element": action_payload},
        originator_user_id=user_id
    )

    # Clear the temporary preview of the user who made the change
    async with canvas_tool.previews_lock:
        if user_id in canvas_tool.active_user_previews.get(canvas_id, {}):
            del canvas_tool.active_user_previews[canvas_id][user_id]

    # MODIFICATION: Call the non-blocking broadcast method.
    canvas_tool._broadcast_to_canvas_listeners(
        canvas_id=canvas_id, event_type="clear_user_preview",
        data={"user_id": user_id}, originator_user_id=user_id
    )

    return Result.ok(info=f"Action '{action_type}' persisted and broadcast.")

markdown_to_svg(self, request, markdown_text='', width=400, font_family='sans-serif', font_size=14, bg_color='#ffffff', text_color='#000000') async

Converts a string of Markdown text into an SVG image. The SVG is returned as a base64 encoded data URL. This version uses a viewBox for better scalability and multi-line handling.

Source code in toolboxv2/mods/Canvas.py

@export(mod_name=MOD_NAME, api=True, version=VERSION, name="markdown_to_svg", api_methods=['POST'],
        request_as_kwarg=True)
async def markdown_to_svg(self, request: RequestData, markdown_text: str = "", width: int = 400,
                          font_family: str = "sans-serif", font_size: int = 14,
                          bg_color: str = "#ffffff", text_color: str = "#000000") -> Result:
    """
    Converts a string of Markdown text into an SVG image.
    The SVG is returned as a base64 encoded data URL.
    This version uses a viewBox for better scalability and multi-line handling.
    """
    if request is None:
        return Result.default_user_error("Request data is missing.", 400)
    if not markdown_text and request.data:
        markdown_text = request.data.get("markdown_text", "")

    if not markdown_text:
        return Result.default_user_error("markdown_text cannot be empty.")

    try:
        # Convert Markdown to HTML
        html_content = markdown2.markdown(markdown_text, extras=["fenced-code-blocks", "tables", "strike"])

        # --- FIX for Multi-line text ---
        # The key is to NOT set a fixed height on the SVG itself, but to use a viewBox.
        # The client will determine the final rendered size.
        # The width of the div inside the foreignObject controls the line wrapping.

        # We still need a rough height for the viewBox.
        # Estimate height: (number of lines * line-height) + padding
        # A simple line-height estimate is font_size * 1.6
        line_height_estimate = font_size * 1.6
        num_lines_estimate = len(html_content.split('\n')) + html_content.count('<br') + html_content.count(
            '<p>') + html_content.count('<li>')
        estimated_height = (num_lines_estimate * line_height_estimate) + 40  # 20px top/bottom padding

        svg_template = f"""
        <svg viewBox="0 0 {width} {int(estimated_height)}" xmlns="http://www.w3.org/2000/svg">
            <foreignObject x="0" y="0" width="{width}" height="{int(estimated_height)}">
                <div xmlns="http://www.w3.org/1999/xhtml">
                    <style>
                        div {{
                            font-family: {font_family};
                            font-size: {font_size}px;
                            color: {text_color};
                            background-color: {bg_color};
                            padding: 10px;
                            border-radius: 5px;
                            line-height: 1.6;
                            width: {width - 20}px; /* Width minus padding */
                            word-wrap: break-word;
                            height: 100%;
                            overflow-y: auto; /* Allow scrolling if content overflows estimate */
                        }}
                        h1, h2, h3 {{ border-bottom: 1px solid #ccc; padding-bottom: 5px; margin-top: 1em; }}
                        pre {{ background-color: #f0f0f0; padding: 10px; border-radius: 4px; overflow-x: auto; }}
                        code {{ font-family: monospace; }}
                        table {{ border-collapse: collapse; width: 100%; }}
                        th, td {{ border: 1px solid #ddd; padding: 8px; }}
                        th {{ background-color: #f2f2f2; }}
                        blockquote {{ border-left: 4px solid #ccc; padding-left: 10px; color: #555; margin-left: 0; }}
                    </style>
                    {html_content}
                </div>
            </foreignObject>
        </svg>
        """

        svg_base64 = base64.b64encode(svg_template.encode('utf-8')).decode('utf-8')
        data_url = f"data:image/svg+xml;base64,{svg_base64}"

        # --- FIX for Editability ---
        # Return the original markdown text along with the SVG
        return Result.ok(data={"svg_data_url": data_url, "original_markdown": markdown_text})

    except Exception as e:
        self.app.logger.error(f"Error converting Markdown to SVG: {e}", exc_info=True)
        return Result.default_internal_error("Failed to convert Markdown to SVG.")

save_session(app, request, data) async

Saves the entire state of a canvas session to the database. This is typically triggered by a user's explicit "Save" action.

Source code in toolboxv2/mods/Canvas.py

@export(mod_name=MOD_NAME, api=True, version=VERSION, name="save_session", api_methods=['POST'], request_as_kwarg=True)
async def save_session(app: App, request: RequestData, data: dict[str, Any] | IdeaSessionData) -> Result:
    """
    Saves the entire state of a canvas session to the database.
    This is typically triggered by a user's explicit "Save" action.
    """
    if not data:
        return Result.default_user_error("Request data is missing.", 400)
    if request is None:
        return Result.default_user_error("Request data is missing.", 400)
    canvas_tool = app.get_mod(MOD_NAME)
    if not canvas_tool or not canvas_tool.db_mod:
        app.logger.error("Save failed: Canvas module or DB not available.")
        return Result.custom_error(info="Database module not available.", exec_code=503)

    user_db_key_base = await canvas_tool._get_user_specific_db_key(request, SESSION_DATA_PREFIX)
    if not user_db_key_base:
        return Result.default_user_error(info="User authentication required to save.", exec_code=401)

    try:
        # Validate the incoming data against the Pydantic model
        session_data_obj = IdeaSessionData(**data) if isinstance(data, dict) else data
    except Exception as e:
        app.logger.error(f"Invalid session data for save: {e}. Data: {str(data)[:500]}", exc_info=True)
        return Result.default_user_error(info=f"Invalid session data format: {e}", exec_code=400)

    # Update timestamp and construct the main session key
    if session_data_obj:
        session_data_obj.last_modified = datetime.now(UTC).timestamp()
    session_db_key = f"{user_db_key_base}_{session_data_obj.id}"

    # Save the full session object to the database
    canvas_tool.db_mod.set(session_db_key, session_data_obj.model_dump_json(exclude_none=True))
    app.logger.info(f"Saved session data for C:{session_data_obj.id}")

    # --- Update the session list metadata ---
    session_list_key = f"{user_db_key_base}{SESSION_LIST_KEY_SUFFIX}"
    try:
        list_res_obj = canvas_tool.db_mod.get(session_list_key)
        user_sessions = []
        if list_res_obj and not list_res_obj.is_error() and list_res_obj.get():
            list_content = list_res_obj.get()[0] if isinstance(list_res_obj.get(), list) else list_res_obj.get()
            user_sessions = json.loads(list_content)

        # Find and update the existing entry, or add a new one
        session_metadata = {
            "id": session_data_obj.id,
            "name": session_data_obj.name,
            "last_modified": session_data_obj.last_modified
        }
        found_in_list = False
        for i, sess_meta in enumerate(user_sessions):
            if sess_meta.get("id") == session_data_obj.id:
                user_sessions[i] = session_metadata
                found_in_list = True
                break
        if not found_in_list:
            user_sessions.append(session_metadata)

        canvas_tool.db_mod.set(session_list_key, json.dumps(user_sessions))
        app.logger.info(f"Updated session list for user key ending in ...{user_db_key_base[-12:]}")

    except Exception as e:
        app.logger.error(f"Failed to update session list for C:{session_data_obj.id}. Error: {e}", exc_info=True)
        # Non-fatal error; the main data was saved. We can continue.

    return Result.ok(
        info="Session saved successfully.",
        data={"id": session_data_obj.id, "last_modified": session_data_obj.last_modified}
    )

ChatModule

get_chat_ui(app)

Liefert das Haupt-HTML-UI für das Chat-Widget. Es verwendet app.web_context(), um das notwendige tbjs CSS und JS einzubinden.

Source code in toolboxv2/mods/ChatModule.py

@export(mod_name=Name, version=version, api=True, name="ui", row=True)
def get_chat_ui(app: App) -> Result:
    """
    Liefert das Haupt-HTML-UI für das Chat-Widget.
    Es verwendet `app.web_context()`, um das notwendige tbjs CSS und JS einzubinden.
    """

    html_content = f"""
        {app.web_context()}
        <style>
            body {{
                display: flex;
                align-items: center;
                justify-content: center;
                min-height: 100vh;
                padding: 1rem;
                background-color: var(--theme-bg);
            }}
        </style>
        <main id="chat-container" style="width: 100%; height: 80vh;">
            <!-- Das Chat-Widget wird hier initialisiert -->
        </main>

        <script unsave="true">
            // Verwende TB.once, um sicherzustellen, dass das Framework vollständig initialisiert ist,
            // bevor unser Code ausgeführt wird.
            TB.once(() => {{
                const chatContainer = document.getElementById('chat-container');
                if (chatContainer && TB.ui.ChatWidget) {{
                    // Initialisiere das Chat-Widget in unserem Container
                    TB.ui.ChatWidget.init(chatContainer);

                    // Verbinde mit dem in diesem Modul definierten WebSocket-Endpunkt
                    TB.ui.ChatWidget.connect();
                }} else {{
                    console.error("Chat UI initialization failed: container or ChatWidget not found.");
                }}
            }});
        </script>
    """

    return Result.html(data=html_content)

on_chat_message(app, conn_id, session, payload) async

Wird aufgerufen, wenn eine Nachricht von einem Client empfangen wird.

Source code in toolboxv2/mods/ChatModule.py

async def on_chat_message(app: App, conn_id: str, session: dict, payload: dict):
    """
    Wird aufgerufen, wenn eine Nachricht von einem Client empfangen wird.
    """
    username = session.get("user_name", "Anonymous")
    print(f"WS MESSAGE from {username} ({conn_id}): {session}")
    message_text = payload.get("data", {}).get("message", "").strip()

    if not message_text:
        return  # Ignoriere leere Nachrichten

    app.print(f"WS MESSAGE from {username} ({conn_id}): {message_text}")

    # Sende die Nachricht an alle im Raum (einschließlich des Absenders)
    await app.ws_broadcast(
        channel_id="ChatModule/public_room",
        payload={"event": "new_message", "data": {"user": username, "text": message_text}}
    )

on_user_connect(app, conn_id, session) async

Wird vom Rust WebSocket Actor aufgerufen, wenn ein neuer Client eine Verbindung herstellt.

Source code in toolboxv2/mods/ChatModule.py

async def on_user_connect(app: App, conn_id: str, session: dict):
    """
    Wird vom Rust WebSocket Actor aufgerufen, wenn ein neuer Client eine Verbindung herstellt.
    """
    username = session.get("user_name", "Anonymous")
    app.print(f"WS CONNECT: User '{username}' connected with conn_id: {conn_id}")

    # Sende eine Willkommensnachricht direkt an den neuen Benutzer (1-zu-1)
    await app.ws_send(conn_id, {"event": "welcome", "data": f"Welcome to the public chat, {username}!"})

    # Kündige den neuen Benutzer allen anderen im Raum an (1-zu-n)
    await app.ws_broadcast(
        channel_id="ChatModule/public_room",
        payload={"event": "user_joined", "data": f"👋 {username} has joined the chat."},
        source_conn_id=conn_id  # Schließt den Absender von diesem Broadcast aus
    )

on_user_disconnect(app, conn_id, session=None) async

Wird aufgerufen, wenn die Verbindung eines Clients geschlossen wird.

Source code in toolboxv2/mods/ChatModule.py

async def on_user_disconnect(app: App, conn_id: str, session: dict=None):
    """
    Wird aufgerufen, wenn die Verbindung eines Clients geschlossen wird.
    """
    if session is None:
        session = {}
    username = session.get("user_name", "Anonymous")
    app.print(f"WS DISCONNECT: User '{username}' disconnected (conn_id: {conn_id})")

    # Kündige den Weggang des Benutzers allen verbleibenden Benutzern im Raum an
    await app.ws_broadcast(
        channel_id="ChatModule/public_room",
        payload={"event": "user_left", "data": f"😥 {username} has left the chat."}
    )

register_chat_handlers(app)

Registriert die asynchronen Funktionen als Handler für spezifische WebSocket-Ereignisse. Der Funktionsname (register_chat_handlers) ist beliebig. Der Decorator ist entscheidend.

Returns:

Type	Description
dict	Ein Dictionary, das Ereignisnamen auf ihre Handler-Funktionen abbildet.

Source code in toolboxv2/mods/ChatModule.py

@export(mod_name=Name, version=version, websocket_handler="public_room")
def register_chat_handlers(app: App) -> dict:
    """
    Registriert die asynchronen Funktionen als Handler für spezifische WebSocket-Ereignisse.
    Der Funktionsname (`register_chat_handlers`) ist beliebig. Der Decorator ist entscheidend.

    Returns:
        Ein Dictionary, das Ereignisnamen auf ihre Handler-Funktionen abbildet.
    """
    return {
        "on_connect": on_user_connect,
        "on_message": on_chat_message,
        "on_disconnect": on_user_disconnect,
    }

CloudM

check_multiple_processes(pids)

Checks the status of multiple processes in a single system call. Returns a dictionary mapping PIDs to their status (GREEN_CIRCLE, RED_CIRCLE, or YELLOW_CIRCLE).

Source code in toolboxv2/mods/CloudM/mini.py

def check_multiple_processes(pids: list[int]) -> dict[int, str]:
    """
    Checks the status of multiple processes in a single system call.
    Returns a dictionary mapping PIDs to their status (GREEN_CIRCLE, RED_CIRCLE, or YELLOW_CIRCLE).
    """
    if not pids:
        return {}

    pid_status = {}

    if os.name == 'nt':  # Windows
        try:
            # Windows tasklist requires separate /FI for each filter
            command = 'tasklist'

            # Add encoding handling for Windows
            result = subprocess.run(
                command,
                capture_output=True,
                text=True,
                shell=True,
                encoding='cp850'  # Use cp850 for Windows console output
            )
            # Create a set of running PIDs from the output
            running_pids = set()
            for line in result.stdout.lower().split('\n'):
                for pid in pids:
                    if str(pid) in line:
                        running_pids.add(pid)
            # Assign status based on whether PID was found in output
            for pid in pids:
                if pid in running_pids:
                    pid_status[pid] = GREEN_CIRCLE
                else:
                    pid_status[pid] = RED_CIRCLE

        except subprocess.SubprocessError as e:
            print(f"SubprocessError: {e}")  # For debugging
            # Mark all as YELLOW_CIRCLE if there's an error running the command
            for pid in pids:
                pid_status[pid] = YELLOW_CIRCLE
        except UnicodeDecodeError as e:
            print(f"UnicodeDecodeError: {e}")  # For debugging
            # Try alternate encoding if cp850 fails
            try:
                result = subprocess.run(
                    command,
                    capture_output=True,
                    text=True,
                    shell=True,
                    encoding='utf-8'
                )
                running_pids = set()
                for line in result.stdout.lower().split('\n'):
                    for pid in pids:
                        if str(pid) in line:
                            running_pids.add(pid)

                for pid in pids:
                    pid_status[pid] = GREEN_CIRCLE if pid in running_pids else RED_CIRCLE
            except Exception as e:
                print(f"Failed with alternate encoding: {e}")  # For debugging
                for pid in pids:
                    pid_status[pid] = YELLOW_CIRCLE

    else:  # Unix/Linux/Mac
        try:
            pids_str = ','.join(str(pid) for pid in pids)
            command = f'ps -p {pids_str} -o pid='

            result = subprocess.run(
                command,
                capture_output=True,
                text=True,
                shell=True,
                encoding='utf-8'
            )
            running_pids = set(int(pid) for pid in result.stdout.strip().split())

            for pid in pids:
                pid_status[pid] = GREEN_CIRCLE if pid in running_pids else RED_CIRCLE

        except subprocess.SubprocessError as e:
            print(f"SubprocessError: {e}")  # For debugging
            for pid in pids:
                pid_status[pid] = YELLOW_CIRCLE

    return pid_status

get_service_pids(info_dir)

Extracts service names and PIDs from pid files.

Source code in toolboxv2/mods/CloudM/mini.py

def get_service_pids(info_dir):
    """Extracts service names and PIDs from pid files."""
    services = {}
    pid_files = [f for f in os.listdir(info_dir) if re.match(r'(.+)-(.+)\.pid', f)]
    for pid_file in pid_files:
        match = re.match(r'(.+)-(.+)\.pid', pid_file)
        if match:
            services_type, service_name = match.groups()
            # Read the PID from the file
            with open(os.path.join(info_dir, pid_file)) as file:
                pid = file.read().strip()
                # Store the PID using a formatted key
                services[f"{service_name} - {services_type}"] = int(pid)
    return services

get_service_status(dir)

Displays the status of all services.

Source code in toolboxv2/mods/CloudM/mini.py

def get_service_status(dir: str) -> str:
    """Displays the status of all services."""
    if time.time()-services_data_sto_last_update_time[0] > 30:
        services = get_service_pids(dir)
        services_data_sto[0] = services
        services_data_sto_last_update_time[0] = time.time()
    else:
        services = services_data_sto[0]
    if not services:
        return "No services found"

    # Get status for all PIDs in a single call
    pid_statuses = check_multiple_processes(list(services.values()))

    # Build the status string
    res_s = "Service(s):" + ("\n" if len(services) > 1 else ' ')
    for service_name, pid in services.items():
        status = pid_statuses.get(pid, YELLOW_CIRCLE)
        res_s += f"{status} {service_name} (PID: {pid})\n"
    services_data_display[0] = res_s.strip()
    return res_s.rstrip()

AuthManager

delete_user(app, username)

Deletes a user and all their data.

Source code in toolboxv2/mods/CloudM/AuthManager.py

@export(mod_name=Name, state=True, test=False, interface=ToolBoxInterfaces.native)
def delete_user(app: App, username: str):
    """Deletes a user and all their data."""
    if not db_helper_test_exist(app, username):
        return Result.default_user_error(f"User '{username}' not found.")

    # This will delete all entries matching the user
    result = db_helper_delete_user(app, username, '*', matching=True)

    if result.is_ok():
        # Also remove the local private key file if it exists
        app.config_fh.remove_key_file_handler("Pk" + Code.one_way_hash(username, "dvp-k")[:8])
        return Result.ok(f"User '{username}' deleted successfully.")
    else:
        return Result.default_internal_error(f"Failed to delete user '{username}'.", data=result)

list_users(app)

Lists all registered users.

Source code in toolboxv2/mods/CloudM/AuthManager.py

@export(mod_name=Name, state=True, test=False, interface=ToolBoxInterfaces.native)
def list_users(app: App):
    """Lists all registered users."""
    keys_result = app.run_any(TBEF.DB.GET, query="all-k", get_results=True)
    if keys_result.is_error():
        return keys_result

    user_keys = keys_result.get()
    if not user_keys:
        return Result.ok("No users found.")

    users = []
    for key in user_keys:
        if isinstance(key, bytes):
            key = key.decode()
        if not key.startswith("USER::"):
            continue
        # Extract username from the key USER::username::uid
        parts = key.split('::')
        if len(parts) > 1 and parts[1] not in [u['username'] for u in users]:
            user_res = get_user_by_name(app, parts[1])
            if user_res.is_ok():
                user_data = user_res.get()
                users.append({"username": user_data.name, "email": user_data.email, "level": user_data.level})

    return Result.ok(data=users)

LogInSystem

cli_logout(app=None) async

Enhanced logout with modern visual feedback

Source code in toolboxv2/mods/CloudM/LogInSystem.py

async def cli_logout(app: App = None):
    """Enhanced logout with modern visual feedback"""
    if app is None:
        app = get_app("CloudM.cli_logout")

    # Clear screen
    print('\033[2J\033[H')

    print_box_header("Logout Process", "🔓")
    print_box_content("Terminating session...", "info")
    print_box_footer()

    username = app.get_username()

    if not username:
        print_status("No active session found", "warning")
        return Result.ok("No session to logout")

    print_status(f"Logging out user: {username}", "progress")
    with Spinner("Finalizing logout..."):
        # Close CLI session
        from .UserInstances import UserInstances, close_cli_session

        try:
            cli_session_id = UserInstances.get_cli_session_id(username).get()
            close_cli_session(cli_session_id)
            print_status("CLI session closed", "success")
        except Exception as e:
            print_status(f"Session cleanup warning: {e}", "warning")

        # Clear JWT token
        from toolboxv2.utils.extras.blobs import BlobFile
        try:
            with BlobFile(f"claim/{username}/jwt.c", key=Code.DK()(), mode="w") as blob:
                blob.clear()
            print_status("Credentials cleared", "success")
        except Exception as e:
            print_status(f"Credential cleanup warning: {e}", "warning")

        # Clear session
        if app.session:
            app.session.valid = False
            app.session.username = None

    print()
    print_box_header("Logout Complete", "✓")
    print_box_content(f"User '{username}' logged out successfully", "success")
    print_box_content("All credentials have been cleared", "info")
    print_box_footer()

    return Result.ok("Logout successful")

cli_web_login(app=None, force_remote=False, force_local=False) async

Enhanced CLI web login with remote/local options and modern visual feedback

Source code in toolboxv2/mods/CloudM/LogInSystem.py

async def cli_web_login(app: App = None, force_remote: bool = False, force_local: bool = False):
    """
    Enhanced CLI web login with remote/local options and modern visual feedback
    """
    if app is None:
        app = get_app("CloudM.cli_web_login")

    # Check if already logged in
    if app.session and app.session.valid:
        print_box_header("Already Authenticated", "✓")
        print_box_content("You are already logged in!", "success")
        print_box_content(f"Username: {app.get_username()}", "info")
        print_box_footer()
        return Result.ok("Already authenticated")

    # Determine login method
    login_method = await _determine_login_method(app, force_remote, force_local)

    if login_method == "remote":
        return await _remote_web_login(app)
    elif login_method == "local":
        return await _local_web_login(app)
    else:
        print_status("Login cancelled by user", "warning")
        return Result.default_user_error("Login cancelled by user")

open_check_cli_auth(session_id, app=None) async

Check if CLI authentication is complete

Source code in toolboxv2/mods/CloudM/LogInSystem.py

@export(mod_name=Name, version=version, api=True)
async def open_check_cli_auth(session_id: str, app: App = None):
    """Check if CLI authentication is complete"""
    if app is None:
        app = get_app("CloudM.open_check_cli_auth")

    # Check session storage for completed authentication
    auth_data_str = await _get_db_helper(app, f"cli_auth_{session_id}")

    if auth_data_str:
        try:
            auth_data = json.loads(auth_data_str)
            return {
                'authenticated': True,
                'jwt_token': auth_data.get('jwt_token'),
                'username': auth_data.get('username')
            }
        except (json.JSONDecodeError, TypeError) as e:
            app.logger.error(f"Error parsing CLI auth data for session {session_id}: {e}")
            return {'authenticated': False}

    return {'authenticated': False}

open_complete_cli_auth(session_id, jwt_token, username, app=None) async

Complete CLI authentication process - No auth required as this IS the auth process

Source code in toolboxv2/mods/CloudM/LogInSystem.py

@export(mod_name=Name, version=version, api=True)
async def open_complete_cli_auth(session_id: str, jwt_token: str, username: str, app: App = None):
    """Complete CLI authentication process - No auth required as this IS the auth process"""
    if app is None:
        app = get_app("CloudM.open_complete_cli_auth")

    app.logger.info(f"CLI auth completion requested for session {session_id}, user {username}")

    # Store authentication data temporarily
    auth_data = {
        'jwt_token': jwt_token,
        'username': username,
        'timestamp': time.time()
    }

    try:
        await _set_db_helper(app, {"query": f"cli_auth_{session_id}", "value": json.dumps(auth_data)})
        app.logger.info(f"CLI auth data stored for session {session_id}")

        # Clean up after 10 minutes
        asyncio.create_task(_cleanup_auth_session(app, session_id, 600))

        return Result.ok("CLI authentication completed")
    except Exception as e:
        app.logger.error(f"Error storing CLI auth data for session {session_id}: {e}")
        return Result.default_internal_error(f"Failed to complete CLI authentication: {str(e)}")

open_web_login_web(app, request, session_id=None, return_to=None, log_in_for='CLI') async

Handle web login flow for CLI and Browser

Source code in toolboxv2/mods/CloudM/LogInSystem.py

@export(mod_name=Name, version=version, api=True, request_as_kwarg=True)
async def open_web_login_web(app: App, request: RequestData, session_id=None, return_to=None, log_in_for="CLI"):
    """Handle web login flow for CLI and Browser"""
    if request is None:
        return Result.default_internal_error("No request specified")

    template = """<div">
        <div class="login-card">
            <h1>🔐 XXX Authentication</h1>
            <p>Please authenticate to continue with XXX access</p>

            <div id="loginForm">
                <input type="text" id="username" placeholder="Username" required>
                <button type="button" onclick="startLogin()">Login</button>
            </div>

            <div id="statusMessage" style="display: none; padding: 10px; margin: 10px 0; border-radius: 5px;"></div>
            <div id="successMessage" style="display: none;">
                <h3>✅ Authentication Successful!</h3>
                <p>You can now return to your XXX. This window will close automatically.</p>
            </div>
        </div>

    <script unsave="true">

        const urlParams = new URLSearchParams(window.location.search);
        const sessionId = urlParams.get('session_id');
        const returnTo = urlParams.get('return_to');

        console.log('[XXX Login] Session ID:', sessionId);
        console.log('[XXX Login] Return to:', returnTo);

        async function startLogin() {
            const username = document.getElementById('username').value;
            if (!username) {
                showStatus('Please enter a username', 'error');
                return;
            }

            if (!sessionId) {
                showStatus('Error: No session ID provided. Please restart the XXX login process.', 'error');
                return;
            }

            showStatus('Authenticating...', 'info');

            try {
                // Use TB.js login system
                console.log('[XXX Login] Attempting login for:', username);
                const result = await window.TB.user.loginWithDeviceKey(username);
                console.log('[XXX Login] Login result:', result);

                if (result.success) {
                    showStatus('Login successful! Notifying XXX...', 'info');

                    // Get the JWT token from TB.user state
                    const jwtToken = window.TB.user.getToken();
                    console.log('[XXX Login] JWT Token:', jwtToken ? 'Found' : 'Not found');

                    if (!jwtToken) {
                        showStatus('Error: No authentication token found. Please try again.', 'error');
                        return;
                    }

                    // Complete XXX authentication by notifying the backend
                    console.log('[XXX Login] Calling complete_cli_auth with session_id:', sessionId);
                    const completeResponse = await window.TB.api.request(
                        'CloudM',
                        'open_complete_cli_auth',
                        {
                            session_id: sessionId,
                            jwt_token: jwtToken,
                            username: username
                        },
                        'POST'
                    );

                    console.log('[XXX Login] complete_cli_auth response:', completeResponse);

                    if (completeResponse.error === window.TB.ToolBoxError.none) {
                        showSuccess();
                        console.log('[XXX Login] XXX authentication completed successfully');
                        setTimeout(() => {
                            console.log('[XXX Login] Closing window...');
                            window.close();
                        }, 3000);
                    } else {
                        showStatus('Error notifying XXX: ' + (completeResponse.info?.help_text || 'Unknown error'), 'error');
                    }
                } else {
                    showStatus(result.message || 'Login failed', 'error');
                }
            } catch (error) {
                console.error('[XXX Login] Error:', error);
                showStatus('Authentication error: ' + error.message, 'error');
            }
        }

        function showStatus(message, type) {
            const statusEl = document.getElementById('statusMessage');
            statusEl.textContent = message;
            statusEl.style.display = 'block';

            // Style based on type
            if (type === 'error') {
                statusEl.style.backgroundColor = '#fee';
                statusEl.style.color = '#c00';
                statusEl.style.border = '1px solid #c00';
            } else if (type === 'info') {
                statusEl.style.backgroundColor = '#eef';
                statusEl.style.color = '#006';
                statusEl.style.border = '1px solid #006';
            } else if (type === 'success') {
                statusEl.style.backgroundColor = '#efe';
                statusEl.style.color = '#060';
                statusEl.style.border = '1px solid #060';
            }
        }

        function showSuccess() {
            document.getElementById('loginForm').style.display = 'none';
            document.getElementById('statusMessage').style.display = 'none';
            document.getElementById('successMessage').style.display = 'block';
        }

        // Auto-focus username field
        if (document.getElementById('username')) {
            document.getElementById('username').focus();

            // Allow Enter key to submit
            document.getElementById('username').addEventListener('keypress', function(e) {
                if (e.key === 'Enter') {
                e.preventDefault();
                    startLogin();
                }
            });
        }

        // Check if TB is available
        if (!window.TB) {
            console.error('[XXX Login] TB framework not loaded!');
            showStatus('Error: TB framework not loaded. Please refresh the page.', 'error');
        } else {
            console.log('[XXX Login] TB framework loaded successfully');
        }


    </script>
</div>
""".replace("XXX", log_in_for)

    return Result.html(template)

print_menu_option(number, text, selected=False)

Print a menu option

Source code in toolboxv2/mods/CloudM/LogInSystem.py

def print_menu_option(number: int, text: str, selected: bool = False):
    """Print a menu option"""
    if selected:
        print(f"  ▶ {number}. {text}")
    else:
        print(f"    {number}. {text}")

ModManager

CloudM - Advanced Module Manager Production-ready module management system with multi-platform support Version: 0.1.0

ConfigVersion

Bases: Enum

Configuration file versions

Source code in toolboxv2/mods/CloudM/ModManager.py

class ConfigVersion(Enum):
    """Configuration file versions"""
    V1 = "1.0"
    V2 = "2.0"

MenuCategory dataclass

Menu category.

Source code in toolboxv2/mods/CloudM/ModManager.py

@dataclass
class MenuCategory:
    """Menu category."""
    name: str
    icon: str
    items: List[MenuItem]

MenuItem dataclass

Menu item with action.

Source code in toolboxv2/mods/CloudM/ModManager.py

@dataclass
class MenuItem:
    """Menu item with action."""
    key: str
    label: str
    action: Callable
    category: str = ""
    icon: str = "•"
    description: str = ""

ModernMenuManager

Modern menu manager with arrow key navigation.

Source code in toolboxv2/mods/CloudM/ModManager.py

class ModernMenuManager:
    """Modern menu manager with arrow key navigation."""

    def __init__(self, app_instance: Optional[Any] = None):
        self.app_instance = app_instance
        self.selected_index = 0
        self.categories: List[MenuCategory] = []
        self.flat_items: List[MenuItem] = []
        self.running = True

    def add_category(self, category: MenuCategory):
        """Add a menu category."""
        self.categories.append(category)
        self.flat_items.extend(category.items)

    def get_menu_text(self) -> List[tuple]:
        """Generate formatted menu text."""
        lines = []

        # Header
        lines.append(('class:menu-border', '╔' + '═' * 68 + '╗\n'))
        lines.append(('class:menu-border', '║'))
        lines.append(('class:menu-title', '  🌩️  CloudM - Module Manager'.center(68)))
        lines.append(('class:menu-border', '║\n'))
        lines.append(('class:menu-border', '╠' + '═' * 68 + '╣\n'))

        # Menu items by category
        current_flat_index = 0

        for cat_idx, category in enumerate(self.categories):
            # Category header
            if cat_idx > 0:
                lines.append(('class:menu-border', '║' + '─' * 68 + '║\n'))

            lines.append(('class:menu-border', '║ '))
            lines.append(('class:menu-category', f'{category.icon} {category.name}'))
            lines.append(('', ' ' * (67 - len(category.name) - len(category.icon)- (2 if len(category.icon) == 1 else 1))))
            lines.append(('class:menu-border', '║\n'))

            # Category items
            for item in category.items:
                is_selected = current_flat_index == self.selected_index

                lines.append(('class:menu-border', '║ '))

                if is_selected:
                    lines.append(('class:menu-item-selected', f' ▶ '))
                else:
                    lines.append(('', '   '))

                # Key
                if is_selected:
                    lines.append(('class:menu-item-selected', f'{item.key:>3}'))
                else:
                    lines.append(('class:menu-key', f'{item.key:>3}'))

                # Label

                if is_selected:
                    lines.append(('class:menu-item-selected', f' {item.icon} {item.label}'))
                    remaining = 60 - len(item.label) - len(item.icon) - (2 if len(item.icon) == 1 else 1)
                    lines.append(('class:menu-item-selected', ' ' * remaining))
                else:
                    lines.append(('class:menu-item', f' {item.icon} {item.label}'))
                    remaining = 60 - len(item.label) - len(item.icon) - (2 if len(item.icon) == 1 else 1)
                    lines.append(('', ' ' * remaining))

                lines.append(('class:menu-border', '║\n'))
                current_flat_index += 1

        # Footer
        lines.append(('class:menu-border', '╚' + '═' * 68 + '╝\n'))
        lines.append(('class:footer', '\n  ↑↓ or w/s: Navigate  │  Enter: Select  │  q: Quit\n'))

        return lines

    def move_up(self):
        """Move selection up."""
        if self.selected_index > 0:
            self.selected_index -= 1

    def move_down(self):
        """Move selection down."""
        if self.selected_index < len(self.flat_items) - 1:
            self.selected_index += 1

    def get_selected_item(self) -> Optional[MenuItem]:
        """Get currently selected menu item."""
        if 0 <= self.selected_index < len(self.flat_items):
            return self.flat_items[self.selected_index]
        return None

    async def run(self):
        """Run the menu manager."""
        # Build menu structure
        self._build_menu()

        while self.running:
            # Clear screen
            print('\033[2J\033[H')

            # Display menu
            menu_text = self.get_menu_text()
            print_formatted_text(FormattedText(menu_text), style=MODERN_STYLE)

            # Key bindings
            kb = KeyBindings()

            @kb.add('up')
            @kb.add('w')
            def move_up_handler(event):
                self.move_up()
                event.app.exit()

            @kb.add('down')
            @kb.add('s')
            def move_down_handler(event):
                self.move_down()
                event.app.exit()

            @kb.add('enter')
            def select_handler(event):
                event.app.exit(result='select')

            @kb.add('q')
            @kb.add('escape')
            def quit_handler(event):
                event.app.exit(result='quit')

            # Wait for input
            dummy_app = Application(
                layout=Layout(Window(FormattedTextControl(''))),
                key_bindings=kb,
                full_screen=False
            )

            result = await dummy_app.run_async()

            if result == 'quit':
                if await show_confirm('Exit Manager', 'Are you sure you want to exit?'):
                    self.running = False
                    break
            elif result == 'select':
                selected = self.get_selected_item()
                if selected:
                    try:
                        await selected.action()
                    except KeyboardInterrupt:
                        continue
                    except Exception as e:
                        await show_message('Error', f'An error occurred:\n\n{str(e)}', 'error')

    def _build_menu(self):
        """Build menu structure with all operations."""

        # =================== MODULE OPERATIONS ===================
        module_ops = MenuCategory(
            name="MODULE OPERATIONS",
            icon="📦",
            items=[
                MenuItem("1", "List all modules", self._list_modules, icon="📋"),
                MenuItem("2", "Install/Update module", self._install_module, icon="📥"),
                MenuItem("3", "Uninstall module", self._uninstall_module, icon="🗑️"),
                MenuItem("4", "Build installer", self._build_installer, icon="🔨"),
                MenuItem("5", "Upload module", self._upload_module, icon="☁️"),
                MenuItem("6", "Update ALL modules", self._update_all, icon="🔄"),
                MenuItem("7", "Build ALL modules", self._build_all, icon="🏗️"),
            ]
        )

        # =================== CONFIGURATION ===================
        config_ops = MenuCategory(
            name="CONFIGURATION",
            icon="⚙️",
            items=[
                MenuItem("8", "View module info", self._view_info, icon="ℹ️"),
                MenuItem("9", "Validate config", self._validate_config, icon="✓ "),
                MenuItem("10", "Create new config", self._create_config, icon="✎ "),
                MenuItem("11", "Generate ALL configs", self._generate_all_configs, icon="⚡"),
                MenuItem("12", "Generate config for module", self._generate_single_config, icon="⚙️"),
            ]
        )

        # =================== PLATFORM & TEMPLATES ===================
        platform_ops = MenuCategory(
            name="PLATFORM & TEMPLATES",
            icon="🌐",
            items=[
                MenuItem("13", "Build platform installer", self._build_platform, icon="🖥️"),
                MenuItem("14", "Install for platform", self._install_platform, icon="💾"),
                MenuItem("15", "Create from template", self._create_from_template, icon="🎨"),
                MenuItem("16", "List templates", self._list_templates, icon="📚"),
            ]
        )

        self.add_category(module_ops)
        self.add_category(config_ops)
        self.add_category(platform_ops)

    # =================== Action Handlers ===================

    async def _list_modules(self):
        """List all modules."""
        await show_progress("Loading Modules", "Scanning module directory...")

        mods = self.app_instance.get_all_mods()

        if not mods:
            await show_message("No Modules", "No modules found in the directory.", "warning")
            return

        # Build module list
        lines = [f"\n{'#':<4} {'Status':<8} {'Module Name':<35} {'Version':<10}"]
        lines.append('─' * 75)

        for i, mod in enumerate(mods, 1):
            mod_obj = self.app_instance.get_mod(mod)
            ver = getattr(mod_obj, 'version', '?.?.?') if mod_obj else '?.?.?'

            # Check config
            config_path = Path('./mods') / mod / 'tbConfig.yaml'
            single_config = Path('./mods') / f'{mod}.yaml'
            status = "✓ OK" if (config_path.exists() or single_config.exists()) else "✗ No cfg"

            lines.append(f"{i:<4} {status:<8} {mod:<35} {ver:<10}")

        lines.append('─' * 75)
        lines.append(f"\nTotal: {len(mods)} modules")

        await show_message(f"📦 Available Modules ({len(mods)})", '\n'.join(lines), "info")

    async def _install_module(self):
        """Install or update a module."""
        module_name = await show_input("Install Module", "Enter module name:")

        if not module_name:
            return

        await show_progress("Installing", f"Installing module '{module_name}'...")

        result = await installer(self.app_instance, module_name)

        if result.is_error:
            await show_message("Installation Failed", f"Error: {result}", "error")
        else:
            await show_message("Success", f"Module '{module_name}' installed successfully!", "success")

    async def _uninstall_module(self):
        """Uninstall a module."""
        module_name = await show_input("Uninstall Module", "Enter module name:")

        if not module_name:
            return

        if not await show_confirm("Confirm Uninstall", f"Really uninstall '{module_name}'?"):
            return

        await show_progress("Uninstalling", f"Removing module '{module_name}'...")

        result = uninstaller(self.app_instance, module_name)

        if result.is_error:
            await show_message("Uninstall Failed", f"Error: {result}", "error")
        else:
            await show_message("Success", f"Module '{module_name}' uninstalled successfully!", "success")

    async def _build_installer(self):
        """Build module installer."""
        module_name = await show_input("Build Installer", "Enter module name:")

        if not module_name:
            return

        upload = await show_confirm("Upload", "Upload after building?")

        await show_progress("Building", f"Building installer for '{module_name}'...")

        result = await make_installer(self.app_instance, module_name, upload=upload)

        if result.is_error:
            await show_message("Build Failed", f"Error: {result}", "error")
        else:
            msg = f"Installer built successfully!"
            if upload:
                msg += "\n\nModule uploaded to cloud!"
            await show_message("Success", msg, "success")

    async def _upload_module(self):
        """Upload module to cloud."""
        module_name = await show_input("Upload Module", "Enter module name:")

        if not module_name:
            return

        await show_progress("Uploading", f"Uploading '{module_name}' to cloud...")

        result = await upload(self.app_instance, module_name)

        if result.is_error:
            await show_message("Upload Failed", f"Error: {result}", "error")
        else:
            await show_message("Success", f"Module '{module_name}' uploaded successfully!", "success")

    async def _update_all(self):
        """Update all modules."""
        if not await show_confirm(
            "Batch Update",
            "This will update ALL modules.\nThis may take several minutes.\n\nContinue?"
        ):
            return

        await show_progress("Batch Update", "Updating all modules... Please wait.")

        result = await update_all_mods(self.app_instance)

        if result.is_error:
            await show_message("Update Completed", f"Completed with errors:\n\n{result}", "warning")
        else:
            await show_message("Success", "All modules updated successfully!", "success")

    async def _build_all(self):
        """Build all modules."""
        upload = await show_confirm("Upload", "Upload after building?")

        if not await show_confirm(
            "Batch Build",
            "This will build ALL modules.\nThis may take several minutes.\n\nContinue?"
        ):
            return

        await show_progress("Batch Build", "Building all modules... Please wait.")

        result = await build_all_mods(self.app_instance, upload=upload)

        if result.is_error:
            await show_message("Build Completed", f"Completed with errors:\n\n{result}", "warning")
        else:
            msg = "All modules built successfully!"
            if upload:
                msg += "\n\nAll modules uploaded to cloud!"
            await show_message("Success", msg, "success")

    async def _view_info(self):
        """View module information."""
        module_name = await show_input("Module Info", "Enter module name:")

        if not module_name:
            return

        await show_progress("Loading", f"Fetching info for '{module_name}'...")

        result = await get_mod_info(self.app_instance, module_name)

        if result.is_error:
            await show_message("Error", f"Could not get module info:\n\n{result}", "error")
        else:
            info_text = yaml.dump(result.get(), default_flow_style=False, allow_unicode=True)
            await show_message(f"Module Info: {module_name}", info_text, "info")

    async def _validate_config(self):
        """Validate module configuration."""
        module_name = await show_input("Validate Config", "Enter module name:")

        if not module_name:
            return

        config_path = Path('./mods') / module_name / 'tbConfig.yaml'
        if not config_path.exists():
            config_path = Path('./mods') / f'{module_name}.yaml'

        if not config_path.exists():
            await show_message("Error", f"Config file not found for '{module_name}'", "error")
            return

        await show_progress("Validating", f"Checking configuration...")

        config, errors = load_and_validate_config(config_path)

        if errors:
            error_text = '\n'.join([f"  {i}. {err}" for i, err in enumerate(errors, 1)])
            await show_message("Validation Failed", f"Errors found:\n\n{error_text}", "error")
        else:
            await show_message("Success", f"Configuration is valid! ✓", "success")

    async def _create_config(self):
        """Create new module configuration."""
        module_name = await show_input("Create Config", "Module name:")
        if not module_name:
            return

        version = await show_input("Version", "Version:", "0.0.1")
        description = await show_input("Description", "Description (optional):")
        author = await show_input("Author", "Author (optional):")

        # Module type selection
        module_type_choice = await show_choice(
            "Module Type",
            "Select module type:",
            [
                ("package", "📦 Package (directory with multiple files)"),
                ("single", "📄 Single (single file module)")
            ]
        )

        if not module_type_choice:
            return

        module_type = ModuleType.SINGLE if module_type_choice == "single" else ModuleType.PACKAGE

        # Create config
        if module_type == ModuleType.PACKAGE:
            config = create_tb_config_v2(
                module_name=module_name,
                version=version,
                module_type=module_type,
                description=description,
                author=author
            )
        else:
            file_path = await show_input("File Path", "Enter file path:")
            if not file_path:
                return

            config = create_tb_config_single(
                module_name=module_name,
                version=version,
                file_path=file_path,
                description=description,
                author=author
            )

        # Save config
        default_path = f"./mods/{module_name}/tbConfig.yaml"
        save_path = await show_input("Save Location", "Save to:", default_path)

        if not save_path:
            return

        try:
            Path(save_path).parent.mkdir(parents=True, exist_ok=True)

            with open(save_path, 'w', encoding='utf-8') as f:
                yaml.dump(config, f, default_flow_style=False, allow_unicode=True)

            await show_message("Success", f"Configuration saved to:\n{save_path}", "success")
        except Exception as e:
            await show_message("Error", f"Could not save config:\n\n{str(e)}", "error")

    async def _generate_all_configs(self):
        """Generate configs for all modules."""
        root_dir = await show_input("Root Directory", "Enter root directory:", "./mods")

        if not root_dir:
            return

        # Generation mode
        mode_choice = await show_choice(
            "Generation Mode",
            "Select generation mode:",
            [
                ("interactive", "💬 Interactive (ask for each module)"),
                ("auto", "🤖 Auto (skip existing configs)"),
                ("force", "⚡ Force (overwrite all configs)")
            ]
        )

        if not mode_choice:
            return

        backup = await show_confirm("Backup", "Create backups of existing configs?")

        interactive_mode = mode_choice == "interactive"
        overwrite_mode = mode_choice == "force"

        if not await show_confirm(
            "Confirm Generation",
            f"Mode: {mode_choice.title()}\n"
            f"Backup: {'Yes' if backup else 'No'}\n"
            f"Root: {root_dir}\n\n"
            "Start generation?"
        ):
            return

        await show_progress("Generating", "Generating configs for all modules...")

        result = await generate_configs_for_existing_mods(
            app=self.app_instance,
            root_dir=root_dir,
            backup=backup,
            interactive=interactive_mode,
            overwrite=overwrite_mode
        )

        if result.is_error:
            await show_message("Completed", f"Generation completed with errors:\n\n{result}", "warning")
        else:
            await show_message("Success", "Config generation completed successfully!", "success")

    async def _generate_single_config(self):
        """Generate config for specific module."""
        # Get module list
        mods = self.app_instance.get_all_mods()

        if not mods:
            await show_message("No Modules", "No modules found.", "warning")
            return

        # Build choices
        choices = []
        for mod in mods:
            config_path = Path('./mods') / mod / 'tbConfig.yaml'
            single_config = Path('./mods') / f'{mod}.yaml'
            status = "✓" if (config_path.exists() or single_config.exists()) else "✗"
            choices.append((mod, f"[{status}] {mod}"))

        module_name = await show_choice(
            "Select Module",
            "Choose module to generate config for:",
            choices
        )

        if not module_name:
            return

        # Check if config exists
        module_path = Path('./mods') / module_name
        config_exists = False

        if module_path.is_dir():
            config_exists = (module_path / 'tbConfig.yaml').exists()
        else:
            config_exists = (Path('./mods') / f'{module_name}.yaml').exists()

        force = False
        if config_exists:
            if not await show_confirm(
                "Config Exists",
                f"Config already exists for '{module_name}'.\n\nOverwrite?"
            ):
                return
            force = True

        await show_progress("Generating", f"Generating config for '{module_name}'...")

        result = await generate_single_module_config(
            app=self.app_instance,
            module_name=module_name,
            force=force
        )

        if result.is_error:
            await show_message("Error", f"Generation failed:\n\n{result}", "error")
        else:
            await show_message("Success", f"Config generated for '{module_name}'!", "success")

    async def _build_platform(self):
        """Build platform-specific installer."""
        module_name = await show_input("Platform Build", "Enter module name:")

        if not module_name:
            return

        # Platform selection
        platform_choices = [(p, f"{p.value}") for p in Platform]
        platform = await show_choice(
            "Select Platform",
            "Choose target platform:",
            platform_choices
        )

        if not platform:
            return

        upload = await show_confirm("Upload", "Upload after building?")

        await show_progress("Building", f"Building for {platform.value}...")

        result = await make_installer(
            self.app_instance, module_name,
            upload=upload,
            platform=platform
        )

        if result.is_error:
            await show_message("Error", f"Build failed:\n\n{result}", "error")
        else:
            await show_message("Success", f"Platform-specific installer built!", "success")

    async def _install_platform(self):
        """Install for specific platform."""
        module_name = await show_input("Platform Install", "Enter module name:")

        if not module_name:
            return

        # Platform selection
        platform_choices = [(p, f"{p.value}") for p in Platform]
        platform = await show_choice(
            "Select Platform",
            "Choose target platform:",
            platform_choices
        )

        if not platform:
            return

        await show_progress("Installing", f"Installing for {platform.value}...")

        result = await installer(self.app_instance, module_name, platform=platform)

        if result.is_error:
            await show_message("Error", f"Installation failed:\n\n{result}", "error")
        else:
            await show_message("Success", "Module installed successfully!", "success")

    async def _create_from_template(self):
        """Create module from template."""
        # Get templates
        result = await list_module_templates(self.app_instance)
        templates = result.get()['templates']

        # Build choices
        template_choices = [
            (t['name'], f"{t['name']:<25} - {t['description']}")
            for t in templates
        ]

        selected_template = await show_choice(
            "Select Template",
            "Choose module template:",
            template_choices
        )

        if not selected_template:
            return

        # Collect information
        module_name = await show_input("Module Name", "Enter module name:")
        if not module_name:
            return

        description = await show_input("Description", "Description (optional):")
        version = await show_input("Version", "Version:", "0.0.1")
        author = await show_input("Author", "Author (optional):")
        location = await show_input("Location", "Location:", "./mods")

        external = await show_confirm("External", "Create external to toolbox?")
        create_config = await show_confirm("Config", "Create tbConfig.yaml?")

        await show_progress("Creating", f"Creating {selected_template} module '{module_name}'...")

        result = await create_module_from_blueprint(
            app=self.app_instance,
            module_name=module_name,
            module_type=selected_template,
            description=description,
            version=version,
            location=location,
            author=author,
            create_config=create_config,
            external=external
        )

        if result.is_error:
            await show_message("Error", f"Module creation failed:\n\n{result}", "error")
        else:
            await show_message(
                "Success",
                f"Module '{module_name}' created successfully!\n\nLocation: {location}/{module_name}",
                "success"
            )

    async def _list_templates(self):
        """List available templates."""
        result = await list_module_templates(self.app_instance)
        templates = result.get()['templates']

        lines = []
        for t in templates:
            lines.append(f"\n┌─ {t['name']}")
            lines.append(f"│  Description: {t['description']}")
            lines.append(f"│  Type: {t['type']}")
            lines.append(f"│  Requires: {', '.join(t['requires']) if t['requires'] else 'None'}")
            lines.append("└" + "─" * 60)

        await show_message("📚 Available Templates", '\n'.join(lines), "info")

add_category(category)

Add a menu category.

Source code in toolboxv2/mods/CloudM/ModManager.py

def add_category(self, category: MenuCategory):
    """Add a menu category."""
    self.categories.append(category)
    self.flat_items.extend(category.items)

get_menu_text()

Generate formatted menu text.

Source code in toolboxv2/mods/CloudM/ModManager.py

def get_menu_text(self) -> List[tuple]:
    """Generate formatted menu text."""
    lines = []

    # Header
    lines.append(('class:menu-border', '╔' + '═' * 68 + '╗\n'))
    lines.append(('class:menu-border', '║'))
    lines.append(('class:menu-title', '  🌩️  CloudM - Module Manager'.center(68)))
    lines.append(('class:menu-border', '║\n'))
    lines.append(('class:menu-border', '╠' + '═' * 68 + '╣\n'))

    # Menu items by category
    current_flat_index = 0

    for cat_idx, category in enumerate(self.categories):
        # Category header
        if cat_idx > 0:
            lines.append(('class:menu-border', '║' + '─' * 68 + '║\n'))

        lines.append(('class:menu-border', '║ '))
        lines.append(('class:menu-category', f'{category.icon} {category.name}'))
        lines.append(('', ' ' * (67 - len(category.name) - len(category.icon)- (2 if len(category.icon) == 1 else 1))))
        lines.append(('class:menu-border', '║\n'))

        # Category items
        for item in category.items:
            is_selected = current_flat_index == self.selected_index

            lines.append(('class:menu-border', '║ '))

            if is_selected:
                lines.append(('class:menu-item-selected', f' ▶ '))
            else:
                lines.append(('', '   '))

            # Key
            if is_selected:
                lines.append(('class:menu-item-selected', f'{item.key:>3}'))
            else:
                lines.append(('class:menu-key', f'{item.key:>3}'))

            # Label

            if is_selected:
                lines.append(('class:menu-item-selected', f' {item.icon} {item.label}'))
                remaining = 60 - len(item.label) - len(item.icon) - (2 if len(item.icon) == 1 else 1)
                lines.append(('class:menu-item-selected', ' ' * remaining))
            else:
                lines.append(('class:menu-item', f' {item.icon} {item.label}'))
                remaining = 60 - len(item.label) - len(item.icon) - (2 if len(item.icon) == 1 else 1)
                lines.append(('', ' ' * remaining))

            lines.append(('class:menu-border', '║\n'))
            current_flat_index += 1

    # Footer
    lines.append(('class:menu-border', '╚' + '═' * 68 + '╝\n'))
    lines.append(('class:footer', '\n  ↑↓ or w/s: Navigate  │  Enter: Select  │  q: Quit\n'))

    return lines

get_selected_item()

Get currently selected menu item.

Source code in toolboxv2/mods/CloudM/ModManager.py

def get_selected_item(self) -> Optional[MenuItem]:
    """Get currently selected menu item."""
    if 0 <= self.selected_index < len(self.flat_items):
        return self.flat_items[self.selected_index]
    return None

move_down()

Move selection down.

Source code in toolboxv2/mods/CloudM/ModManager.py

def move_down(self):
    """Move selection down."""
    if self.selected_index < len(self.flat_items) - 1:
        self.selected_index += 1

move_up()

Move selection up.

Source code in toolboxv2/mods/CloudM/ModManager.py

def move_up(self):
    """Move selection up."""
    if self.selected_index > 0:
        self.selected_index -= 1

run() async

Run the menu manager.

Source code in toolboxv2/mods/CloudM/ModManager.py

async def run(self):
    """Run the menu manager."""
    # Build menu structure
    self._build_menu()

    while self.running:
        # Clear screen
        print('\033[2J\033[H')

        # Display menu
        menu_text = self.get_menu_text()
        print_formatted_text(FormattedText(menu_text), style=MODERN_STYLE)

        # Key bindings
        kb = KeyBindings()

        @kb.add('up')
        @kb.add('w')
        def move_up_handler(event):
            self.move_up()
            event.app.exit()

        @kb.add('down')
        @kb.add('s')
        def move_down_handler(event):
            self.move_down()
            event.app.exit()

        @kb.add('enter')
        def select_handler(event):
            event.app.exit(result='select')

        @kb.add('q')
        @kb.add('escape')
        def quit_handler(event):
            event.app.exit(result='quit')

        # Wait for input
        dummy_app = Application(
            layout=Layout(Window(FormattedTextControl(''))),
            key_bindings=kb,
            full_screen=False
        )

        result = await dummy_app.run_async()

        if result == 'quit':
            if await show_confirm('Exit Manager', 'Are you sure you want to exit?'):
                self.running = False
                break
        elif result == 'select':
            selected = self.get_selected_item()
            if selected:
                try:
                    await selected.action()
                except KeyboardInterrupt:
                    continue
                except Exception as e:
                    await show_message('Error', f'An error occurred:\n\n{str(e)}', 'error')

ModuleType

Bases: Enum

Module types for different installation strategies

Source code in toolboxv2/mods/CloudM/ModManager.py

class ModuleType(Enum):
    """Module types for different installation strategies"""
    PACKAGE = "package"  # Full module directory
    SINGLE = "single"  # Single file module
    HYBRID = "hybrid"  # Mix of both

Platform

Bases: Enum

Supported platform types for module installation

Source code in toolboxv2/mods/CloudM/ModManager.py

class Platform(Enum):
    """Supported platform types for module installation"""
    SERVER = "server"
    CLIENT = "client"
    DESKTOP = "desktop"
    MOBILE = "mobile"
    COMMON = "common"  # Files needed on all platforms
    ALL = "all"

build_all_mods(app, base='mods', upload=True) async

Builds installer packages for all modules.

Parameters:

Name	Type	Description	Default
app	Optional[App]	Application instance	required
base	str	Base directory containing modules	'mods'
upload	bool	Whether to upload packages after building	True

Returns:

Type	Description
Result	Result with build summary

Source code in toolboxv2/mods/CloudM/ModManager.py

@export(mod_name=Name, name="build_all", test=False)
async def build_all_mods(app: Optional[App], base: str = "mods",
                         upload: bool = True) -> Result:
    """
    Builds installer packages for all modules.

    Args:
        app: Application instance
        base: Base directory containing modules
        upload: Whether to upload packages after building

    Returns:
        Result with build summary
    """
    if app is None:
        app = get_app(f"{Name}.build_all")

    all_mods = app.get_all_mods()
    results = {"success": [], "failed": []}

    async def build_pipeline(mod_name: str):
        try:
            result = await make_installer(app, mod_name, os.path.join('.', base), upload)
            if result.is_error:
                results["failed"].append({"module": mod_name, "reason": str(result)})
            else:
                results["success"].append(mod_name)
            return result
        except Exception as e:
            results["failed"].append({"module": mod_name, "reason": str(e)})
            return Result.default_internal_error(str(e))

    # Build all modules
    build_results = [await build_pipeline(mod) for mod in all_mods]

    return Result.ok({
        "summary": {
            "total": len(all_mods),
            "success": len(results["success"]),
            "failed": len(results["failed"])
        },
        "details": results
    })

create_and_pack_module(path, module_name='', version='-.-.-', additional_dirs=None, yaml_data=None, platform_filter=None)

Creates and packs a module into a ZIP file with platform-specific support.

Parameters:

Name	Type	Description	Default
path	str	Path to module directory or file	required
module_name	str	Name of the module	''
version	str	Module version	'-.-.-'
additional_dirs	Optional[Dict]	Additional directories to include	None
yaml_data	Optional[Dict]	Configuration data override	None
platform_filter	Optional[Platform]	Optional platform filter for packaging	None

Returns:

Type	Description
Optional[str]	Path to created ZIP file or None on failure

Source code in toolboxv2/mods/CloudM/ModManager.py

def create_and_pack_module(path: str, module_name: str = '', version: str = '-.-.-',
                           additional_dirs: Optional[Dict] = None,
                           yaml_data: Optional[Dict] = None,
                           platform_filter: Optional[Platform] = None) -> Optional[str]:
    """
    Creates and packs a module into a ZIP file with platform-specific support.

    Args:
        path: Path to module directory or file
        module_name: Name of the module
        version: Module version
        additional_dirs: Additional directories to include
        yaml_data: Configuration data override
        platform_filter: Optional platform filter for packaging

    Returns:
        Path to created ZIP file or None on failure
    """
    if additional_dirs is None:
        additional_dirs = {}
    if yaml_data is None:
        yaml_data = {}

    os.makedirs("./mods_sto/temp/", exist_ok=True)

    module_path = Path(path) / module_name

    if not module_path.exists():
        module_path = Path(f"{path}/{module_name}.py")

    temp_dir = Path(tempfile.mkdtemp(dir="./mods_sto/temp"))

    platform_suffix = f"_{platform_filter.value}" if platform_filter else ""
    zip_file_name = f"RST${module_name}&{__version__}§{version}{platform_suffix}.zip"
    zip_path = Path(f"./mods_sto/{zip_file_name}")

    if not module_path.exists():
        print(f"Module path does not exist: {module_path}")
        return None

    try:
        if module_path.is_dir():
            # Package module - create v2 config
            config_data = create_tb_config_v2(
                module_name=module_name,
                version=version,
                **yaml_data
            )

            config_path = module_path / "tbConfig.yaml"
            with open(config_path, 'w', encoding='utf-8') as f:
                yaml.dump(config_data, f, default_flow_style=False, allow_unicode=True)

            # Generate requirements
            req_path = module_path / "requirements.txt"
            generate_requirements(str(module_path), str(req_path))

            # Copy module directory
            shutil.copytree(module_path, temp_dir / module_path.name, dirs_exist_ok=True)

        else:
            # Single file module - create single config
            config_data = create_tb_config_single(
                module_name=module_name,
                version=version,
                file_path=str(module_path),
                **yaml_data
            )

            # Copy file
            shutil.copy2(module_path, temp_dir)

            # Create config
            config_path = temp_dir / f"{module_name}.yaml"
            with open(config_path, 'w', encoding='utf-8') as f:
                yaml.dump(config_data, f, default_flow_style=False, allow_unicode=True)

            # Generate requirements
            req_path = temp_dir / "requirements.txt"
            generate_requirements(str(temp_dir), str(req_path))

        # Add additional directories
        for dir_name, dir_paths in additional_dirs.items():
            if isinstance(dir_paths, str):
                dir_paths = [dir_paths]

            for dir_path in dir_paths:
                dir_path = Path(dir_path)
                full_path = temp_dir / dir_name

                if dir_path.is_dir():
                    shutil.copytree(dir_path, full_path, dirs_exist_ok=True)
                elif dir_path.is_file():
                    full_path.mkdir(parents=True, exist_ok=True)
                    shutil.copy2(dir_path, full_path)
                else:
                    print(f"Path is neither directory nor file: {dir_path}")

        # Create ZIP file
        with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf:
            for root, _dirs, files in os.walk(temp_dir):
                for file in files:
                    file_path = Path(root) / file
                    arcname = file_path.relative_to(temp_dir)
                    zipf.write(file_path, arcname)

        # Cleanup temporary directory
        shutil.rmtree(temp_dir)

        print(f"✓ Successfully created: {zip_path}")
        return str(zip_path)

    except Exception as e:
        print(f"✗ Error creating module package: {str(e)}")
        if temp_dir.exists():
            shutil.rmtree(temp_dir)
        return None

create_module_from_blueprint(app=None, module_name='', module_type='basic', description='', version='0.0.1', location='./mods', author='', create_config=True, external=False) async

Creates a new module from blueprint template.

Parameters:

Name	Type	Description	Default
app	Optional[App]	Application instance	None
module_name	str	Name of the new module	''
module_type	str	Type of module (basic, async_service, workflow, etc.)	'basic'
description	str	Module description	''
version	str	Initial version	'0.0.1'
location	str	Where to create the module	'./mods'
author	str	Module author	''
create_config	bool	Whether to create tbConfig.yaml	True
external	bool	If True, create external to toolbox structure	False

Returns:

Type	Description
Result	Result with creation status

Source code in toolboxv2/mods/CloudM/ModManager.py

@export(mod_name=Name, name="create_module", test=False)
async def create_module_from_blueprint(
    app: Optional[App] = None,
    module_name: str = "",
    module_type: str = "basic",
    description: str = "",
    version: str = "0.0.1",
    location: str = "./mods",
    author: str = "",
    create_config: bool = True,
    external: bool = False
) -> Result:
    """
    Creates a new module from blueprint template.

    Args:
        app: Application instance
        module_name: Name of the new module
        module_type: Type of module (basic, async_service, workflow, etc.)
        description: Module description
        version: Initial version
        location: Where to create the module
        author: Module author
        create_config: Whether to create tbConfig.yaml
        external: If True, create external to toolbox structure

    Returns:
        Result with creation status
    """
    if app is None:
        app = get_app(f"{Name}.create_module")

    if not module_name:
        return Result.default_user_error("Module name is required")

    if module_type not in MODULE_TEMPLATES:
        return Result.default_user_error(
            f"Invalid module type. Available: {', '.join(MODULE_TEMPLATES.keys())}"
        )

    template = MODULE_TEMPLATES[module_type]

    # Prepare paths
    location_path = Path(location)

    if template["type"] == "package":
        module_path = location_path / module_name
        module_file = module_path / "__init__.py"
    else:
        module_path = location_path
        module_file = module_path / f"{module_name}.py"

    # Check if module already exists
    if module_file.exists():
        return Result.default_user_error(f"Module already exists: {module_file}")

    try:
        # Create directory structure
        if template["type"] == "package":
            module_path.mkdir(parents=True, exist_ok=True)
            print(f"✓ Created package directory: {module_path}")
        else:
            module_path.mkdir(parents=True, exist_ok=True)
            print(f"✓ Using directory: {module_path}")

        # Generate module content
        content = template["content"].format(
            MODULE_NAME=module_name,
            MODULE_NAME_LOWER=module_name.lower(),
            VERSION=version,
            DESCRIPTION=description or template["description"]
        )

        # Write module file
        with open(module_file, 'w', encoding='utf-8') as f:
            f.write(content)
        print(f"✓ Created module file: {module_file}")

        # Create requirements.txt
        req_path = module_path if template["type"] == "package" else module_path
        requirements = []

        if "async" in template["requires"]:
            requirements.append("aiohttp>=3.8.0")

        if requirements:
            req_file = (module_path if template["type"] == "package" else module_path) / "requirements.txt"
            with open(req_file, 'w', encoding='utf-8') as f:
                f.write('\n'.join(requirements))
            print(f"✓ Created requirements.txt")

        # Create tbConfig.yaml
        if create_config and not external:
            if template["type"] == "package":
                config = create_tb_config_v2(
                    module_name=module_name,
                    version=version,
                    module_type=ModuleType.PACKAGE,
                    description=description or template["description"],
                    author=author,
                    metadata={
                        "template": module_type,
                        "created_at": time.strftime("%Y-%m-%d %H:%M:%S")
                    }
                )
                config_path = module_path / "tbConfig.yaml"
            else:
                config = create_tb_config_single(
                    module_name=module_name,
                    version=version,
                    file_path=str(module_file.relative_to(location_path.parent)),
                    description=description or template["description"],
                    author=author,
                    metadata={
                        "template": module_type,
                        "created_at": time.strftime("%Y-%m-%d %H:%M:%S")
                    }
                )
                config_path = module_path / f"{module_name}.yaml"

            with open(config_path, 'w', encoding='utf-8') as f:
                yaml.dump(config, f, default_flow_style=False, allow_unicode=True, sort_keys=False)
            print(f"✓ Created config: {config_path}")

        # Create additional files based on type
        if module_type == "api_endpoint":
            # Create example API documentation
            api_doc = module_path / "API.md" if template["type"] == "package" else module_path / f"{module_name}_API.md"
            with open(api_doc, 'w', encoding='utf-8') as f:
                f.write(f"""# {module_name} API Documentation

## Endpoints

### GET /api/{module_name}/get_items
Get list of items with pagination.

**Query Parameters:**
- `limit` (int, optional): Number of items (default: 10)
- `offset` (int, optional): Pagination offset (default: 0)

**Response:**
```json
{{
  "items": [...],
  "total": 100,
  "limit": 10,
  "offset": 0
}}
```

### POST /api/{module_name}/create_item
Create a new item.

**Request Body:**
```json
{{
  "name": "Item name",
  "description": "Item description"
}}
```

### GET /api/{module_name}/health_check
Health check endpoint.
""")
            print(f"✓ Created API documentation")

        elif module_type == "websocket":
            # Create WebSocket client example
            ws_example = module_path / "client_example.html" if template[
                                                                    "type"] == "package" else module_path / f"{module_name}_client.html"
            with open(ws_example, 'w', encoding='utf-8') as f:
                f.write(f"""<!DOCTYPE html>
<html>
<head>
    <title>{module_name} WebSocket Client</title>
    <script src="/static/tbjs/tb.js"></script>
</head>
<body>
    <h1>{module_name} WebSocket Demo</h1>
    <div id="messages"></div>
    <input type="text" id="messageInput" placeholder="Type a message...">
    <button onclick="sendMessage()">Send</button>

    <script>
        // Connect to WebSocket
        TB.ws.connect('/ws/{module_name}/main', {{
            onOpen: () => {{
                console.log('Connected to {module_name}');
            }},
            onMessage: (data) => {{
                console.log('Message:', data);
                displayMessage(data);
            }}
        }});

        // Listen for specific events
        TB.events.on('ws:event:new_message', ({{ data }}) => {{
            displayMessage(data.data);
        }});

        function sendMessage() {{
            const input = document.getElementById('messageInput');
            TB.ws.send({{
                event: 'message',
                data: {{
                    text: input.value,
                    timestamp: new Date().toISOString()
                }}
            }});
            input.value = '';
        }}

        function displayMessage(msg) {{
            const div = document.getElementById('messages');
            div.innerHTML += `<div>${{JSON.stringify(msg)}}</div>`;
        }}
    </script>
</body>
</html>
""")
            print(f"✓ Created WebSocket client example")

        # Create README
        readme_path = module_path / "README.md" if template[
                                                       "type"] == "package" else module_path / f"{module_name}_README.md"
        with open(readme_path, 'w', encoding='utf-8') as f:
            f.write(f"""# {module_name}

{description or template['description']}

## Version
{version}

## Type
{template['description']}

## Installation

```bash
# Install module
python CloudM.py install {module_name}
```

## Usage

```python
from toolboxv2 import get_app

app = get_app("{module_name}.Example")

# Use module functions
# Example code here
```

## Author
{author or 'ToolBoxV2'}

## Created
{time.strftime("%Y-%m-%d %H:%M:%S")}

## Template
{module_type}
""")
        print(f"✓ Created README.md")

        print(f"\n{'=' * 60}")
        print(f"✓ Module '{module_name}' created successfully!")
        print(f"{'=' * 60}")
        print(f"\nLocation: {module_file}")
        print(f"Type: {template['description']}")
        print(f"Version: {version}")

        if not external:
            print(f"\nNext steps:")
            print(f"1. Review and customize the generated code")
            print(f"2. Install dependencies: pip install -r requirements.txt")
            print(
                f"3. Test the module: python -c 'from toolboxv2 import get_app; app = get_app(\"{module_name}.Test\")'")
            print(f"4. Build installer: python CloudM.py build {module_name}")

        return Result.ok(data={
            "module_name": module_name,
            "type": module_type,
            "location": str(module_file),
            "config_created": create_config,
            "files_created": [
                str(module_file),
                str(readme_path)
            ]
        })

    except Exception as e:
        return Result.default_internal_error(f"Failed to create module: {str(e)}")

create_tb_config_single(module_name, version, file_path, description='', author='', specification=None, dependencies=None, platforms=None, metadata=None)

Creates configuration for single-file modules.

Parameters:

Name	Type	Description	Default
module_name	str	Name of the module	required
version	str	Module version	required
file_path	str	Path to the single file	required
description	str	Module description	''
author	str	Module author	''
specification	Optional[Dict]	File specifications (exports, functions, etc.)	None
dependencies	Optional[List]	List of dependencies	None
platforms	Optional[List[Platform]]	List of supported platforms	None
metadata	Optional[Dict]	Additional metadata	None

Returns:

Type	Description
Dict	Configuration dictionary for single file module

Source code in toolboxv2/mods/CloudM/ModManager.py

def create_tb_config_single(module_name: str, version: str, file_path: str,
                            description: str = "", author: str = "",
                            specification: Optional[Dict] = None,
                            dependencies: Optional[List] = None,
                            platforms: Optional[List[Platform]] = None,
                            metadata: Optional[Dict] = None) -> Dict:
    """
    Creates configuration for single-file modules.

    Args:
        module_name: Name of the module
        version: Module version
        file_path: Path to the single file
        description: Module description
        author: Module author
        specification: File specifications (exports, functions, etc.)
        dependencies: List of dependencies
        platforms: List of supported platforms
        metadata: Additional metadata

    Returns:
        Configuration dictionary for single file module
    """
    if specification is None:
        specification = {
            "exports": [],
            "functions": [],
            "classes": [],
            "requires": []
        }

    if dependencies is None:
        dependencies = []

    if platforms is None:
        platforms = [Platform.ALL.value]
    else:
        platforms = [p.value if isinstance(p, Platform) else p for p in platforms]

    if metadata is None:
        metadata = {}

    return {
        "version": version,
        "config_version": ConfigVersion.V2.value,
        "module_name": module_name,
        "module_type": ModuleType.SINGLE.value,
        "file_path": file_path,
        "description": description,
        "author": author,
        "license": "MIT",
        "specification": specification,
        "dependencies": dependencies,
        "platforms": platforms,
        "metadata": metadata
    }

create_tb_config_v2(module_name, version, module_type=ModuleType.PACKAGE, description='', author='', license='MIT', homepage='', platforms=None, metadata=None)

Creates a v2 tbConfig with platform-specific file management.

Parameters:

Name	Type	Description	Default
module_name	str	Name of the module	required
version	str	Module version	required
module_type	ModuleType	Type of module (package/single/hybrid)	PACKAGE
description	str	Module description	''
author	str	Module author	''
license	str	Module license	'MIT'
homepage	str	Module homepage/repository	''
platforms	Optional[Dict]	Platform-specific file configurations	None
metadata	Optional[Dict]	Additional metadata	None

Returns:

Type	Description
Dict	Configuration dictionary

Source code in toolboxv2/mods/CloudM/ModManager.py

def create_tb_config_v2(module_name: str, version: str, module_type: ModuleType = ModuleType.PACKAGE,
                        description: str = "", author: str = "", license: str = "MIT",
                        homepage: str = "", platforms: Optional[Dict] = None,
                        metadata: Optional[Dict] = None) -> Dict:
    """
    Creates a v2 tbConfig with platform-specific file management.

    Args:
        module_name: Name of the module
        version: Module version
        module_type: Type of module (package/single/hybrid)
        description: Module description
        author: Module author
        license: Module license
        homepage: Module homepage/repository
        platforms: Platform-specific file configurations
        metadata: Additional metadata

    Returns:
        Configuration dictionary
    """
    if platforms is None:
        platforms = {
            Platform.COMMON.value: {"files": ["*"], "required": True},
            Platform.SERVER.value: {"files": [], "required": False},
            Platform.CLIENT.value: {"files": [], "required": False},
            Platform.DESKTOP.value: {"files": [], "required": False},
            Platform.MOBILE.value: {"files": [], "required": False}
        }

    if metadata is None:
        metadata = {}

    return {
        "version": version,
        "config_version": ConfigVersion.V2.value,
        "module_name": module_name,
        "module_type": module_type.value,
        "description": description,
        "author": author,
        "license": license,
        "homepage": homepage,
        "dependencies_file": f"./mods/{module_name}/requirements.txt",
        "zip": f"RST${module_name}&{__version__}§{version}.zip",
        "platforms": platforms,
        "metadata": metadata
    }

download_files(urls, directory, desc, print_func, filename=None)

Downloads files from URLs with progress indication.

Parameters:

Name	Type	Description	Default
urls	List[str]	List of URLs to download	required
directory	str	Target directory	required
desc	str	Progress bar description	required
print_func	callable	Function for printing messages	required
filename	Optional[str]	Optional filename (uses basename if None)	None

Returns:

Type	Description
str	Path to last downloaded file

Source code in toolboxv2/mods/CloudM/ModManager.py

def download_files(urls: List[str], directory: str, desc: str,
                   print_func: callable, filename: Optional[str] = None) -> str:
    """
    Downloads files from URLs with progress indication.

    Args:
        urls: List of URLs to download
        directory: Target directory
        desc: Progress bar description
        print_func: Function for printing messages
        filename: Optional filename (uses basename if None)

    Returns:
        Path to last downloaded file
    """
    for url in tqdm(urls, desc=desc):
        if filename is None:
            filename = os.path.basename(url)
        print_func(f"Downloading {filename}")
        print_func(f"{url} -> {directory}/{filename}")
        os.makedirs(directory, exist_ok=True)
        urllib.request.urlretrieve(url, f"{directory}/{filename}")
    return f"{directory}/{filename}"

download_mod(app, module_name, platform=None) async

Downloads a module ZIP file.

Parameters:

Name	Type	Description	Default
app	App	Application instance	required
module_name	str	Name of module to download	required
platform	Optional[str]	Optional platform filter	None

Returns:

Type	Description
Result	Binary result with ZIP file

Source code in toolboxv2/mods/CloudM/ModManager.py

@export(mod_name=Name, name="download_mod", api=True, api_methods=['GET'])
async def download_mod(app: App, module_name: str,
                       platform: Optional[str] = None) -> Result:
    """
    Downloads a module ZIP file.

    Args:
        app: Application instance
        module_name: Name of module to download
        platform: Optional platform filter

    Returns:
        Binary result with ZIP file
    """
    try:
        zip_path_str = find_highest_zip_version(module_name)

        if not zip_path_str:
            return Result.default_user_error(
                f"Module '{module_name}' not found",
                exec_code=404
            )

        zip_path = Path(zip_path_str)

        if not zip_path.exists():
            return Result.default_user_error(
                f"Module file not found: {zip_path}",
                exec_code=404
            )

        return Result.binary(
            data=zip_path.read_bytes(),
            content_type="application/zip",
            download_name=zip_path.name
        )

    except Exception as e:
        return Result.default_internal_error(f"Download failed: {str(e)}")

format_status(status, message)

Format status message with icon.

Source code in toolboxv2/mods/CloudM/ModManager.py

def format_status(status: str, message: str) -> HTML:
    """Format status message with icon."""
    icons = {
        'success': '✓',
        'error': '✗',
        'warning': '⚠',
        'info': 'ℹ'
    }
    icon = icons.get(status, '•')
    return HTML(f'<{status}>{icon} {message}</{status}>')

generate_configs_for_existing_mods(app=None, root_dir='./mods', backup=True, interactive=True, overwrite=False) async

Generates tbConfig.yaml files for all existing modules in the mods directory.

Supports: - Package modules (directories) -> tbConfig.yaml (v2) - Single file modules (.py files) -> {module_name}.yaml (single)

Parameters:

Name	Type	Description	Default
app	Optional[App]	Application instance	None
root_dir	str	Root directory containing modules	'./mods'
backup	bool	Create backups of existing configs	True
interactive	bool	Ask for confirmation before each operation	True
overwrite	bool	Overwrite existing configs without asking	False

Returns:

Type	Description
Result	Result with generation summary

Source code in toolboxv2/mods/CloudM/ModManager.py

@export(mod_name=Name, name="generate_configs", test=False)
async def generate_configs_for_existing_mods(
    app: Optional[App] = None,
    root_dir: str = './mods',
    backup: bool = True,
    interactive: bool = True,
    overwrite: bool = False
) -> Result:
    """
    Generates tbConfig.yaml files for all existing modules in the mods directory.

    Supports:
    - Package modules (directories) -> tbConfig.yaml (v2)
    - Single file modules (.py files) -> {module_name}.yaml (single)

    Args:
        app: Application instance
        root_dir: Root directory containing modules
        backup: Create backups of existing configs
        interactive: Ask for confirmation before each operation
        overwrite: Overwrite existing configs without asking

    Returns:
        Result with generation summary
    """
    if app is None:
        app = get_app(f"{Name}.generate_configs")

    root_path = Path(root_dir)
    if not root_path.exists():
        return Result.default_user_error(f"Directory not found: {root_dir}")

    results = {
        "generated": [],
        "skipped": [],
        "failed": [],
        "backed_up": []
    }

    def create_backup(config_path: Path) -> bool:
        """Creates a backup of existing config file"""
        if not config_path.exists():
            return False

        backup_path = config_path.with_suffix('.yaml.backup')
        counter = 1
        while backup_path.exists():
            backup_path = config_path.with_suffix(f'.yaml.backup{counter}')
            counter += 1

        shutil.copy2(config_path, backup_path)
        results["backed_up"].append(str(backup_path))
        print(f"  📦 Backup created: {backup_path.name}")
        return True

    def read_requirements(module_path: Path) -> List[str]:
        """Reads dependencies from requirements.txt"""
        req_file = module_path / 'requirements.txt' if module_path.is_dir() else module_path.parent / 'requirements.txt'

        if not req_file.exists():
            return []

        try:
            with open(req_file, 'r', encoding='utf-8') as f:
                return [line.strip() for line in f if line.strip() and not line.startswith('#')]
        except Exception as e:
            print(f"  ⚠ Error reading requirements: {e}")
            return []

    def extract_module_info(module_path: Path, module_name: str) -> Dict[str, Any]:
        """Extracts metadata from module by analyzing the code"""
        info = {
            "version": "0.0.1",
            "description": f"Module {module_name}",
            "author": "",
            "exports": [],
            "dependencies": []
        }

        try:
            # Try to load module to get version
            if module_name in app.get_all_mods():
                mod = app.get_mod(module_name)
                if mod:
                    info["version"] = getattr(mod, 'version', '0.0.1')

            # Analyze Python file for exports
            py_file = module_path if module_path.is_file() else module_path / '__init__.py'
            if not py_file.exists() and module_path.is_dir():
                py_file = module_path / f"{module_name}.py"

            if py_file.exists():
                with open(py_file, 'r', encoding='utf-8') as f:
                    content = f.read()

                    # Extract version
                    import re
                    version_match = re.search(r'version\s*=\s*["\']([^"\']+)["\']', content)
                    if version_match:
                        info["version"] = version_match.group(1)

                    # Extract exports
                    export_matches = re.findall(r'@export\([^)]*name=["\']([^"\']+)["\']', content)
                    info["exports"] = export_matches

                    # Extract docstring
                    docstring_match = re.search(r'"""([^"]+)"""', content)
                    if docstring_match:
                        info["description"] = docstring_match.group(1).strip().split('\n')[0]

            # Get dependencies
            info["dependencies"] = read_requirements(module_path)

        except Exception as e:
            print(f"  ⚠ Error extracting info: {e}")

        return info

    def generate_package_config(module_path: Path, module_name: str) -> bool:
        """Generates tbConfig.yaml for package modules"""
        config_path = module_path / "tbConfig.yaml"

        # Check if config exists
        if config_path.exists() and not overwrite:
            if interactive:
                response = input(f"  Config exists for {module_name}. Overwrite? (y/n/b=backup): ").lower()
                if response == 'n':
                    results["skipped"].append(module_name)
                    print(f"  ⏭  Skipped: {module_name}")
                    return False
                elif response == 'b':
                    create_backup(config_path)
            else:
                results["skipped"].append(module_name)
                print(f"  ⏭  Skipped (exists): {module_name}")
                return False
        elif config_path.exists() and backup:
            create_backup(config_path)

        # Extract module information
        info = extract_module_info(module_path, module_name)

        # Determine platform files
        platform_config = {
            Platform.COMMON.value: {
                "files": ["*"],
                "required": True
            },
            Platform.SERVER.value: {
                "files": [],
                "required": False
            },
            Platform.CLIENT.value: {
                "files": [],
                "required": False
            },
            Platform.DESKTOP.value: {
                "files": [],
                "required": False
            },
            Platform.MOBILE.value: {
                "files": [],
                "required": False
            }
        }

        # Create config
        config = create_tb_config_v2(
            module_name=module_name,
            version=info["version"],
            module_type=ModuleType.PACKAGE,
            description=info["description"],
            author=info["author"],
            platforms=platform_config,
            metadata={
                "exports": info["exports"],
                "auto_generated": True,
                "generated_at": time.strftime("%Y-%m-%d %H:%M:%S")
            }
        )

        # Write config
        try:
            with open(config_path, 'w', encoding='utf-8') as f:
                yaml.dump(config, f, default_flow_style=False, allow_unicode=True, sort_keys=False)

            # Generate requirements.txt if not exists
            req_path = module_path / "requirements.txt"
            if not req_path.exists():
                generate_requirements(str(module_path), str(req_path))

            results["generated"].append(module_name)
            print(f"  ✓ Generated: {config_path}")
            return True

        except Exception as e:
            results["failed"].append({"module": module_name, "error": str(e)})
            print(f"  ✗ Failed: {module_name} - {e}")
            return False

    def generate_single_config(file_path: Path, module_name: str) -> bool:
        """Generates {module_name}.yaml for single file modules"""
        config_path = file_path.parent / f"{module_name}.yaml"

        # Check if config exists
        if config_path.exists() and not overwrite:
            if interactive:
                response = input(f"  Config exists for {module_name}. Overwrite? (y/n/b=backup): ").lower()
                if response == 'n':
                    results["skipped"].append(module_name)
                    print(f"  ⏭  Skipped: {module_name}")
                    return False
                elif response == 'b':
                    create_backup(config_path)
            else:
                results["skipped"].append(module_name)
                print(f"  ⏭  Skipped (exists): {module_name}")
                return False
        elif config_path.exists() and backup:
            create_backup(config_path)

        # Extract module information
        info = extract_module_info(file_path, module_name)

        # Create single config
        config = create_tb_config_single(
            module_name=module_name,
            version=info["version"],
            file_path=str(file_path.relative_to(root_path.parent)),
            description=info["description"],
            author=info["author"],
            specification={
                "exports": info["exports"],
                "functions": [],
                "classes": [],
                "requires": info["dependencies"]
            },
            dependencies=info["dependencies"],
            platforms=[Platform.ALL.value],
            metadata={
                "auto_generated": True,
                "generated_at": time.strftime("%Y-%m-%d %H:%M:%S")
            }
        )

        # Write config
        try:
            with open(config_path, 'w', encoding='utf-8') as f:
                yaml.dump(config, f, default_flow_style=False, allow_unicode=True, sort_keys=False)

            results["generated"].append(module_name)
            print(f"  ✓ Generated: {config_path}")
            return True

        except Exception as e:
            results["failed"].append({"module": module_name, "error": str(e)})
            print(f"  ✗ Failed: {module_name} - {e}")
            return False

    # Main processing loop
    print(f"\n🔍 Scanning directory: {root_path}")
    print("=" * 60)

    items = sorted(root_path.iterdir())
    total_items = len(items)

    for idx, item in enumerate(items, 1):
        # Skip hidden files/folders and __pycache__
        if item.name.startswith('.') or item.name == '__pycache__':
            continue

        print(f"\n[{idx}/{total_items}] Processing: {item.name}")

        if item.is_dir():
            # Package module
            module_name = item.name
            generate_package_config(item, module_name)

        elif item.is_file() and item.suffix == '.py':
            # Single file module
            module_name = item.stem
            generate_single_config(item, module_name)

    # Summary
    print("\n" + "=" * 60)
    print("📊 Generation Summary:")
    print(f"  ✓ Generated: {len(results['generated'])}")
    print(f"  ⏭  Skipped:   {len(results['skipped'])}")
    print(f"  ✗ Failed:    {len(results['failed'])}")
    print(f"  📦 Backed up: {len(results['backed_up'])}")

    if results['generated']:
        print("\n✓ Generated configs for:")
        for mod in results['generated']:
            print(f"  - {mod}")

    if results['failed']:
        print("\n✗ Failed to generate configs for:")
        for fail in results['failed']:
            print(f"  - {fail['module']}: {fail['error']}")

    return Result.ok({
        "summary": {
            "total_processed": total_items,
            "generated": len(results['generated']),
            "skipped": len(results['skipped']),
            "failed": len(results['failed']),
            "backed_up": len(results['backed_up'])
        },
        "details": results
    })

generate_single_module_config(app=None, module_name='', force=False) async

Generates config for a single specific module.

Parameters:

Name	Type	Description	Default
app	Optional[App]	Application instance	None
module_name	str	Name of module to generate config for	''
force	bool	Force overwrite without asking	False

Returns:

Type	Description
Result	Result with generation status

Source code in toolboxv2/mods/CloudM/ModManager.py

@export(mod_name=Name, name="generate_single_config", test=False)
async def generate_single_module_config(
    app: Optional[App] = None,
    module_name: str = "",
    force: bool = False
) -> Result:
    """
    Generates config for a single specific module.

    Args:
        app: Application instance
        module_name: Name of module to generate config for
        force: Force overwrite without asking

    Returns:
        Result with generation status
    """
    if app is None:
        app = get_app(f"{Name}.generate_single_config")

    if not module_name:
        return Result.default_user_error("Module name is required")

    # Find module path
    module_path = Path('./mods') / module_name

    if not module_path.exists():
        # Try as single file
        module_path = Path(f'./mods/{module_name}.py')
        if not module_path.exists():
            return Result.default_user_error(f"Module not found: {module_name}")

    print(f"\n🔧 Generating config for: {module_name}")

    # Use the main function with specific parameters
    result = await generate_configs_for_existing_mods(
        app=app,
        root_dir=str(module_path.parent),
        backup=True,
        interactive=not force,
        overwrite=force
    )

    return result

get_mod_info(app, module_name) async

Gets detailed information about a module.

Parameters:

Name	Type	Description	Default
app	App	Application instance	required
module_name	str	Name of module	required

Returns:

Type	Description
Result	Result with module information

Source code in toolboxv2/mods/CloudM/ModManager.py

@export(mod_name=Name, name="getModInfo", api=True, api_methods=['GET'])
async def get_mod_info(app: App, module_name: str) -> Result:
    """
    Gets detailed information about a module.

    Args:
        app: Application instance
        module_name: Name of module

    Returns:
        Result with module information
    """
    try:
        zip_path = find_highest_zip_version(module_name)

        if not zip_path:
            return Result.default_user_error(
                f"Module '{module_name}' not found",
                exec_code=404
            )

        # Extract and read config
        with zipfile.ZipFile(zip_path, 'r') as zf:
            config_files = [f for f in zf.namelist() if f.endswith('tbConfig.yaml') or f.endswith('.yaml')]

            if not config_files:
                return Result.default_user_error("No configuration file found in module")

            config_content = zf.read(config_files[0])
            config = yaml.safe_load(config_content)

        return Result.ok(config)

    except Exception as e:
        return Result.default_internal_error(f"Failed to get module info: {str(e)}")

get_mod_version(app, module_name) async

Gets the latest version of a module.

Parameters:

Name	Type	Description	Default
app	App	Application instance	required
module_name	str	Name of module	required

Returns:

Type	Description
Result	Result with version string

Source code in toolboxv2/mods/CloudM/ModManager.py

@export(mod_name=Name, name="getModVersion", api=True, api_methods=['GET'])
async def get_mod_version(app: App, module_name: str) -> Result:
    """
    Gets the latest version of a module.

    Args:
        app: Application instance
        module_name: Name of module

    Returns:
        Result with version string
    """
    try:
        version_str = find_highest_zip_version(module_name, version_only=True)

        if version_str:
            return Result.text(version_str)

        return Result.default_user_error(
            f"No build found for module '{module_name}'",
            exec_code=404
        )

    except Exception as e:
        return Result.default_internal_error(f"Failed to get version: {str(e)}")

get_platform_files(config, platform)

Extracts file list for specific platform from config.

Parameters:

Name	Type	Description	Default
config	Dict	Module configuration dictionary	required
platform	Platform	Target platform	required

Returns:

Type	Description
List[str]	List of files for the platform

Source code in toolboxv2/mods/CloudM/ModManager.py

def get_platform_files(config: Dict, platform: Platform) -> List[str]:
    """
    Extracts file list for specific platform from config.

    Args:
        config: Module configuration dictionary
        platform: Target platform

    Returns:
        List of files for the platform
    """
    platforms = config.get("platforms", {})

    # Get common files (required on all platforms)
    common_files = platforms.get(Platform.COMMON.value, {}).get("files", [])

    # Get platform-specific files
    platform_files = platforms.get(platform.value, {}).get("files", [])

    return common_files + platform_files

increment_version(version_str, max_value=99)

Increments a version number in the format "vX.Y.Z".

Parameters:

Name	Type	Description	Default
version_str	str	Current version number (e.g., "v0.0.1")	required
max_value	int	Maximum number per position (default: 99)	99

Returns:

Type	Description
str	Incremented version number

Raises:

Type	Description
ValueError	If version format is invalid

Source code in toolboxv2/mods/CloudM/ModManager.py

def increment_version(version_str: str, max_value: int = 99) -> str:
    """
    Increments a version number in the format "vX.Y.Z".

    Args:
        version_str: Current version number (e.g., "v0.0.1")
        max_value: Maximum number per position (default: 99)

    Returns:
        Incremented version number

    Raises:
        ValueError: If version format is invalid
    """
    if not version_str.startswith("v"):
        raise ValueError("Version must start with 'v' (e.g., 'v0.0.1')")

    version_core = version_str[1:]
    try:
        parsed_version = Version(version_core)
    except ValueError as e:
        raise ValueError(f"Invalid version number: {version_core}") from e

    parts = list(parsed_version.release)

    # Increment rightmost position
    for i in range(len(parts) - 1, -1, -1):
        if parts[i] < max_value:
            parts[i] += 1
            break
        else:
            parts[i] = 0
    else:
        # All positions at max_value, add new position
        parts.insert(0, 1)

    return "v" + ".".join(map(str, parts))

install_dependencies(yaml_file, auto=False)

Installs dependencies from tbConfig.yaml.

Parameters:

Name	Type	Description	Default
yaml_file	str	Path to configuration file	required
auto	bool	Automatically install without confirmation	False

Returns:

Type	Description
bool	True if successful, False otherwise

Source code in toolboxv2/mods/CloudM/ModManager.py

def install_dependencies(yaml_file: str, auto: bool = False) -> bool:
    """
    Installs dependencies from tbConfig.yaml.

    Args:
        yaml_file: Path to configuration file
        auto: Automatically install without confirmation

    Returns:
        True if successful, False otherwise
    """
    try:
        with open(yaml_file, 'r', encoding='utf-8') as f:
            config = yaml.safe_load(f)

        dependencies_file = config.get("dependencies_file")

        if not dependencies_file:
            print("⚠ No dependencies file specified")
            return True

        dependencies_path = Path(dependencies_file)

        if not dependencies_path.exists():
            print(f"⚠ Dependencies file not found: {dependencies_path}")
            return False

        print(f"Installing dependencies from: {dependencies_path}")

        if not auto:
            response = input("Continue with installation? (y/n): ")
            if response.lower() != 'y':
                print("Installation cancelled")
                return False

        subprocess.run(
            [sys.executable, '-m', 'pip', 'install', '-r', str(dependencies_path)],
            check=True
        )

        print("✓ Dependencies installed successfully")
        return True

    except Exception as e:
        print(f"✗ Error installing dependencies: {str(e)}")
        return False

install_from_zip(app, zip_name, no_dep=True, auto_dep=False, target_platform=None)

Installs a module from ZIP file with dependency management.

Parameters:

Name	Type	Description	Default
app	App	Application instance	required
zip_name	str	Name of ZIP file	required
no_dep	bool	Skip dependency installation	True
auto_dep	bool	Automatically install dependencies	False
target_platform	Optional[Platform]	Optional platform filter	None

Returns:

Type	Description
bool	True if successful, False otherwise

Source code in toolboxv2/mods/CloudM/ModManager.py

def install_from_zip(app: App, zip_name: str, no_dep: bool = True,
                     auto_dep: bool = False,
                     target_platform: Optional[Platform] = None) -> bool:
    """
    Installs a module from ZIP file with dependency management.

    Args:
        app: Application instance
        zip_name: Name of ZIP file
        no_dep: Skip dependency installation
        auto_dep: Automatically install dependencies
        target_platform: Optional platform filter

    Returns:
        True if successful, False otherwise
    """
    zip_path = Path(app.start_dir) / "mods_sto" / zip_name

    if not zip_path.exists():
        print(f"✗ ZIP file not found: {zip_path}")
        return False

    try:
        with Spinner(f"Unpacking {zip_path.name[-40:]}"):
            module_name = unpack_and_move_module(
                str(zip_path),
                f"{app.start_dir}/mods",
                target_platform=target_platform
            )

        if not module_name:
            return False

        # Install dependencies if requested
        if not no_dep:
            config_path = Path(app.start_dir) / "mods" / module_name / "tbConfig.yaml"

            if config_path.exists():
                with Spinner(f"Installing dependencies for {module_name}"):
                    install_dependencies(str(config_path), auto_dep)

        return True

    except Exception as e:
        print(f"✗ Installation failed: {str(e)}")
        return False

installer(app, module_name, build_state=True, platform=None) async

Installs or updates a module from the server.

Parameters:

Name	Type	Description	Default
app	Optional[App]	Application instance	required
module_name	str	Name of module to install	required
build_state	bool	Whether to rebuild state after installation	True
platform	Optional[Platform]	Optional platform filter for installation	None

Returns:

Type	Description
Result	Result with installation status

Source code in toolboxv2/mods/CloudM/ModManager.py

@export(mod_name=Name, name="install", test=False)
async def installer(app: Optional[App], module_name: str,
                    build_state: bool = True,
                    platform: Optional[Platform] = None) -> Result:
    """
    Installs or updates a module from the server.

    Args:
        app: Application instance
        module_name: Name of module to install
        build_state: Whether to rebuild state after installation
        platform: Optional platform filter for installation

    Returns:
        Result with installation status
    """
    if app is None:
        app = get_app(f"{Name}.install")

    if not app.session.valid and not await app.session.login():
        return Result.default_user_error("Please login with CloudM")

    try:
        # Get remote version
        response = await app.session.fetch(
            f"/api/{Name}/getModVersion?module_name={module_name}",
            method="GET"
        )
        remote_version = await response.text()
        remote_version = None if remote_version == "None" else remote_version.strip('"')

        # Get local version
        local_version = find_highest_zip_version(module_name, version_only=True)

        if not local_version and not remote_version:
            return Result.default_user_error(f"Module '{module_name}' not found (404)")

        # Compare versions
        local_ver = pv.parse(local_version) if local_version else pv.parse("0.0.0")
        remote_ver = pv.parse(remote_version) if remote_version else pv.parse("0.0.0")

        app.print(f"Module versions - Local: {local_ver}, Remote: {remote_ver}")

        if remote_ver > local_ver:
            download_path = Path(app.start_dir) / 'mods_sto'
            download_url = f"/api/{Name}/download_mod?module_name={module_name}"

            if platform:
                download_url += f"&platform={platform.value}"

            app.print(f"Downloading from {app.session.base}{download_url}")

            if not await app.session.download_file(download_url, str(download_path)):
                app.print("⚠ Automatic download failed")
                manual = input("Download manually and place in mods_sto folder. Done? (y/n): ")
                if 'y' not in manual.lower():
                    return Result.default_user_error("Installation cancelled")

            zip_name = f"RST${module_name}&{app.version}§{remote_version}.zip"

            with Spinner("Installing from ZIP"):
                success = install_from_zip(app, zip_name, target_platform=platform)

            if not success:
                return Result.default_internal_error("Installation failed")

            if build_state:
                with Spinner("Rebuilding state"):
                    get_state_from_app(app)

            return Result.ok({
                "message": f"Module '{module_name}' installed successfully",
                "version": remote_version
            })

        app.print("✓ Module is already up to date")
        return Result.ok("Module is up to date")

    except Exception as e:
        return Result.default_internal_error(f"Installation failed: {str(e)}")

interactive_manager(app=None) async

Modern interactive CLI manager for module operations.

Features: - Arrow key navigation (↑↓ or w/s) - Modern, minimalistic UI - All original functionality preserved - Better visual feedback - Elegant dialogs and prompts

Source code in toolboxv2/mods/CloudM/ModManager.py

@export(mod_name=Name, name="manager", test=False)
async def interactive_manager(app: Optional[App] = None):
    """
    Modern interactive CLI manager for module operations.

    Features:
    - Arrow key navigation (↑↓ or w/s)
    - Modern, minimalistic UI
    - All original functionality preserved
    - Better visual feedback
    - Elegant dialogs and prompts
    """
    if app is None:
        app = get_app(f"{Name}.manager")

    # Clear screen
    print('\033[2J\033[H')

    # Welcome message
    print_formatted_text(HTML(
        '\n<menu-title>╔════════════════════════════════════════════════════════════════════╗</menu-title>\n'
        '<menu-title>║          Welcome to CloudM Interactive Module Manager              ║</menu-title>\n'
        '<menu-title>╚════════════════════════════════════════════════════════════════════╝</menu-title>\n'
    ), style=MODERN_STYLE)

    await asyncio.sleep(1)

    # Create and run manager
    manager = ModernMenuManager(app)

    try:
        await manager.run()
    except KeyboardInterrupt:
        pass
    finally:
        # Goodbye message
        print('\033[2J\033[H')
        print_formatted_text(HTML(
            '\n<success>╔════════════════════════════════════════════════════════════════════╗</success>\n'
            '<success>║          Thank you for using CloudM Module Manager! 👋             ║</success>\n'
            '<success>╚════════════════════════════════════════════════════════════════════╝</success>\n'
        ), style=MODERN_STYLE)

list_module_templates(app=None)

Lists all available module templates.

Returns:

Type	Description
Result	Result with template information

Source code in toolboxv2/mods/CloudM/ModManager.py

@export(mod_name=Name, name="list_templates", test=False)
def list_module_templates(app: Optional[App] = None) -> Result:
    """
    Lists all available module templates.

    Returns:
        Result with template information
    """
    templates = []
    for template_name, template_info in MODULE_TEMPLATES.items():
        templates.append({
            "name": template_name,
            "description": template_info["description"],
            "type": template_info["type"],
            "requires": template_info["requires"]
        })

    return Result.ok(data={"templates": templates, "count": len(templates)})

list_modules(app=None)

Lists all available modules.

Returns:

Type	Description
Result	Result with module list

Source code in toolboxv2/mods/CloudM/ModManager.py

@export(mod_name=Name, api=True, interface=ToolBoxInterfaces.remote, test=False)
def list_modules(app: App = None) -> Result:
    """
    Lists all available modules.

    Returns:
        Result with module list
    """
    if app is None:
        app = get_app("cm.list_modules")

    modules = app.get_all_mods()
    return Result.ok({"modules": modules, "count": len(modules)})

load_and_validate_config(config_path)

Loads and validates a configuration file.

Parameters:

Name	Type	Description	Default
config_path	Path	Path to configuration file	required

Returns:

Type	Description
Tuple[Optional[Dict], List[str]]	Tuple of (config_dict or None, list_of_errors)

Source code in toolboxv2/mods/CloudM/ModManager.py

def load_and_validate_config(config_path: Path) -> Tuple[Optional[Dict], List[str]]:
    """
    Loads and validates a configuration file.

    Args:
        config_path: Path to configuration file

    Returns:
        Tuple of (config_dict or None, list_of_errors)
    """
    if not config_path.exists():
        return None, [f"Config file not found: {config_path}"]

    try:
        with open(config_path, 'r', encoding='utf-8') as f:
            config = yaml.safe_load(f)
    except Exception as e:
        return None, [f"Failed to parse YAML: {str(e)}"]

    # Determine schema based on module_type
    module_type = config.get("module_type", "package")

    if module_type == "single":
        schema = TB_CONFIG_SINGLE_SCHEMA
    else:
        schema = TB_CONFIG_SCHEMA_V2

    is_valid, errors = validate_config(config, schema)

    if not is_valid:
        return config, errors

    return config, []

make_installer(app, module_name, base='./mods', upload=None, platform=None) async

Creates an installer package for a module.

Parameters:

Name	Type	Description	Default
app	Optional[App]	Application instance	required
module_name	str	Name of module to package	required
base	str	Base directory containing modules	'./mods'
upload	Optional[bool]	Whether to upload after creation	None
platform	Optional[Platform]	Optional platform filter	None

Returns:

Type	Description
Result	Result with package path or upload status

Source code in toolboxv2/mods/CloudM/ModManager.py

@export(mod_name=Name, name="make_install", test=False)
async def make_installer(app: Optional[App], module_name: str,
                         base: str = "./mods", upload: Optional[bool] = None,
                         platform: Optional[Platform] = None) -> Result:
    """
    Creates an installer package for a module.

    Args:
        app: Application instance
        module_name: Name of module to package
        base: Base directory containing modules
        upload: Whether to upload after creation
        platform: Optional platform filter

    Returns:
        Result with package path or upload status
    """
    if app is None:
        app = get_app(f"{Name}.make_install")

    if module_name not in app.get_all_mods():
        return Result.default_user_error(f"Module '{module_name}' not found")

    try:
        with Spinner("Testing module load"):
            app.save_load(module_name)

        mod = app.get_mod(module_name)
        version_ = getattr(mod, 'version', version)

        with Spinner("Creating and packing module"):
            zip_path = create_and_pack_module(
                base, module_name, version_,
                platform_filter=platform
            )

        if not zip_path:
            return Result.default_internal_error("Failed to create package")

        # Upload if requested
        if upload or (upload is None and 'y' in input("Upload ZIP file? (y/n): ").lower()):
            with Spinner("Uploading file"):
                res = await app.session.upload_file(zip_path, '/installer/upload-file/')

            if isinstance(res, dict):
                if res.get('res', '').startswith('Successfully uploaded'):
                    return Result.ok({
                        "message": "Module packaged and uploaded",
                        "zip_path": zip_path,
                        "upload_response": res
                    })
                return Result.default_user_error(res)

        return Result.ok({
            "message": "Module packaged successfully",
            "zip_path": zip_path
        })

    except Exception as e:
        return Result.default_internal_error(f"Installation creation failed: {str(e)}")

mod_manager_ui(app)

Serves the module manager web interface.

Returns:

Type	Description
Result	HTML result with UI

Source code in toolboxv2/mods/CloudM/ModManager.py

@export(mod_name=Name, name="ui", api=True, api_methods=['GET'])
def mod_manager_ui(app: App) -> Result:
    """
    Serves the module manager web interface.

    Returns:
        HTML result with UI
    """
    ui_path = Path(__file__).parent / "mod_manager.html"

    if not ui_path.exists():
        # Generate default UI if file doesn't exist
        html_content = """
<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>CloudM - Module Manager</title>
    <style>
        * { margin: 0; padding: 0; box-sizing: border-box; }
        body {
            font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
            background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
            min-height: 100vh;
            padding: 20px;
        }
        .container {
            max-width: 1200px;
            margin: 0 auto;
            background: white;
            border-radius: 10px;
            box-shadow: 0 20px 60px rgba(0,0,0,0.3);
            overflow: hidden;
        }
        .header {
            background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
            color: white;
            padding: 30px;
            text-align: center;
        }
        .content {
            padding: 30px;
        }
        .module-list {
            display: grid;
            grid-template-columns: repeat(auto-fill, minmax(300px, 1fr));
            gap: 20px;
            margin-top: 20px;
        }
        .module-card {
            border: 1px solid #e0e0e0;
            border-radius: 8px;
            padding: 20px;
            transition: transform 0.2s, box-shadow 0.2s;
        }
        .module-card:hover {
            transform: translateY(-5px);
            box-shadow: 0 5px 20px rgba(0,0,0,0.1);
        }
        .btn {
            background: #667eea;
            color: white;
            border: none;
            padding: 10px 20px;
            border-radius: 5px;
            cursor: pointer;
            margin: 5px;
            transition: background 0.3s;
        }
        .btn:hover { background: #5568d3; }
        .btn-danger { background: #e74c3c; }
        .btn-danger:hover { background: #c0392b; }
    </style>
</head>
<body>
    <div class="container">
        <div class="header">
            <h1>🚀 CloudM Module Manager</h1>
            <p>Manage your modules with ease</p>
        </div>
        <div class="content">
            <div class="controls">
                <button class="btn" onclick="loadModules()">🔄 Refresh</button>
                <button class="btn" onclick="updateAll()">⬆️ Update All</button>
            </div>
            <div id="modules" class="module-list"></div>
        </div>
    </div>
    <script>
        async function loadModules() {
            const response = await fetch('/api/CloudM/list_modules');
            const data = await response.json();
            const container = document.getElementById('modules');
            container.innerHTML = data.modules.map(mod => `
                <div class="module-card">
                    <h3>📦 ${mod}</h3>
                    <button class="btn" onclick="installModule('${mod}')">Install</button>
                    <button class="btn btn-danger" onclick="uninstallModule('${mod}')">Uninstall</button>
                </div>
            `).join('');
        }
        async function installModule(name) {
            alert(`Installing ${name}...`);
        }
        async function uninstallModule(name) {
            if (confirm(`Uninstall ${name}?`)) {
                alert(`Uninstalling ${name}...`);
            }
        }
        async function updateAll() {
            alert('Updating all modules...');
        }
        loadModules();
    </script>
</body>
</html>
        """
        return Result.html(html_content)

    return Result.html(ui_path.read_text(encoding='utf-8'))

run_command(command, cwd=None)

Executes a command and returns output.

Parameters:

Name	Type	Description	Default
command	List[str]	Command and arguments as list	required
cwd	Optional[str]	Working directory for command execution	None

Returns:

Type	Description
str	Command stdout output

Raises:

Type	Description
CalledProcessError	If command fails

Source code in toolboxv2/mods/CloudM/ModManager.py

def run_command(command: List[str], cwd: Optional[str] = None) -> str:
    """
    Executes a command and returns output.

    Args:
        command: Command and arguments as list
        cwd: Working directory for command execution

    Returns:
        Command stdout output

    Raises:
        subprocess.CalledProcessError: If command fails
    """
    result = subprocess.run(
        command,
        cwd=cwd,
        capture_output=True,
        text=True,
        check=True,
        encoding='utf-8'
    )
    return result.stdout

show_choice(title, text, choices) async

Show radio list dialog.

Source code in toolboxv2/mods/CloudM/ModManager.py

async def show_choice(title: str, text: str, choices: List[tuple]) -> Optional[Any]:
    """Show radio list dialog."""
    result = await radiolist_dialog(
        title=f"◉ {title}",
        text=text,
        values=choices,
        style=MODERN_STYLE
    ).run_async()
    return result

show_confirm(title, text) async

Show confirmation dialog.

Source code in toolboxv2/mods/CloudM/ModManager.py

async def show_confirm(title: str, text: str) -> bool:
    """Show confirmation dialog."""
    result = await yes_no_dialog(
        title=f"⚠ {title}",
        text=text,
        style=MODERN_STYLE
    ).run_async()
    return result if result is not None else False

show_input(title, label, default='') async

Show input dialog.

Source code in toolboxv2/mods/CloudM/ModManager.py

async def show_input(title: str, label: str, default: str = "") -> Optional[str]:
    """Show input dialog."""
    result = await input_dialog(
        title=f"✎ {title}",
        text=label,
        default=default,
        style=MODERN_STYLE
    ).run_async()
    return result

show_message(title, text, style='info') async

Show a message dialog.

Source code in toolboxv2/mods/CloudM/ModManager.py

async def show_message(title: str, text: str, style: str = "info"):
    """Show a message dialog."""
    icons = {
        'success': '✓',
        'error': '✗',
        'warning': '⚠',
        'info': 'ℹ'
    }
    icon = icons.get(style, 'ℹ')

    await message_dialog(
        title=f"{icon} {title}",
        text=text,
        style=MODERN_STYLE
    ).run_async()

show_progress(title, message) async

Show a simple progress message.

Source code in toolboxv2/mods/CloudM/ModManager.py

async def show_progress(title: str, message: str):
    """Show a simple progress message."""
    print_formatted_text(HTML(f'\n<info>⟳ {title}</info>'))
    print_formatted_text(HTML(f'<menu-item>  {message}</menu-item>\n'))

uninstall_dependencies(yaml_file)

Uninstalls dependencies from tbConfig.yaml.

Parameters:

Name	Type	Description	Default
yaml_file	str	Path to configuration file	required

Returns:

Type	Description
bool	True if successful, False otherwise

Source code in toolboxv2/mods/CloudM/ModManager.py

def uninstall_dependencies(yaml_file: str) -> bool:
    """
    Uninstalls dependencies from tbConfig.yaml.

    Args:
        yaml_file: Path to configuration file

    Returns:
        True if successful, False otherwise
    """
    try:
        with open(yaml_file, 'r', encoding='utf-8') as f:
            config = yaml.safe_load(f)

        dependencies = config.get("dependencies", [])

        if not dependencies:
            print("⚠ No dependencies to uninstall")
            return True

        for dependency in dependencies:
            print(f"Uninstalling: {dependency}")
            subprocess.run(
                [sys.executable, '-m', 'pip', 'uninstall', '-y', dependency],
                check=True
            )

        print("✓ Dependencies uninstalled successfully")
        return True

    except Exception as e:
        print(f"✗ Error uninstalling dependencies: {str(e)}")
        return False

uninstall_module(path, module_name='', version='-.-.-', additional_dirs=None, yaml_data=None)

Uninstalls a module by removing its directory and ZIP file.

Parameters:

Name	Type	Description	Default
path	str	Base path containing module	required
module_name	str	Name of module to uninstall	''
version	str	Module version	'-.-.-'
additional_dirs	Optional[Dict]	Additional directories to remove	None
yaml_data	Optional[Dict]	Configuration data	None

Returns:

Type	Description
bool	True if successful, False otherwise

Source code in toolboxv2/mods/CloudM/ModManager.py

def uninstall_module(path: str, module_name: str = '', version: str = '-.-.-',
                     additional_dirs: Optional[Dict] = None,
                     yaml_data: Optional[Dict] = None) -> bool:
    """
    Uninstalls a module by removing its directory and ZIP file.

    Args:
        path: Base path containing module
        module_name: Name of module to uninstall
        version: Module version
        additional_dirs: Additional directories to remove
        yaml_data: Configuration data

    Returns:
        True if successful, False otherwise
    """
    if additional_dirs is None:
        additional_dirs = {}

    base_path = Path(path).parent
    module_path = base_path / module_name
    zip_path = Path(f"./mods_sto/RST${module_name}&{__version__}§{version}.zip")

    if not module_path.exists():
        print(f"⚠ Module {module_name} already uninstalled")
        return False

    try:
        # Remove module directory
        shutil.rmtree(module_path)
        print(f"✓ Removed module directory: {module_path}")

        # Remove additional directories
        for _dir_name, dir_paths in additional_dirs.items():
            if isinstance(dir_paths, str):
                dir_paths = [dir_paths]

            for dir_path in dir_paths:
                dir_path = Path(dir_path)
                if dir_path.exists():
                    shutil.rmtree(dir_path)
                    print(f"✓ Removed additional path: {dir_path}")

        # Remove ZIP file
        if zip_path.exists():
            zip_path.unlink()
            print(f"✓ Removed ZIP file: {zip_path}")

        return True

    except Exception as e:
        print(f"✗ Error during uninstallation: {str(e)}")
        return False

uninstaller(app, module_name)

Uninstalls a module.

Parameters:

Name	Type	Description	Default
app	Optional[App]	Application instance	required
module_name	str	Name of module to uninstall	required

Returns:

Type	Description
Result	Result with uninstallation status

Source code in toolboxv2/mods/CloudM/ModManager.py

@export(mod_name=Name, name="uninstall", test=False)
def uninstaller(app: Optional[App], module_name: str) -> Result:
    """
    Uninstalls a module.

    Args:
        app: Application instance
        module_name: Name of module to uninstall

    Returns:
        Result with uninstallation status
    """
    if app is None:
        app = get_app(f"{Name}.uninstall")

    if module_name not in app.get_all_mods():
        return Result.default_user_error(f"Module '{module_name}' not found")

    try:
        mod = app.get_mod(module_name)
        version_ = getattr(mod, 'version', version)

        confirm = input(f"Uninstall module '{module_name}' v{version_}? (y/n): ")
        if 'y' not in confirm.lower():
            return Result.ok("Uninstallation cancelled")

        success = uninstall_module(f"./mods/{module_name}", module_name, version_)

        if success:
            return Result.ok(f"Module '{module_name}' uninstalled successfully")
        else:
            return Result.default_internal_error("Uninstallation failed")

    except Exception as e:
        return Result.default_internal_error(f"Uninstallation failed: {str(e)}")

unpack_and_move_module(zip_path, base_path='./mods', module_name='', target_platform=None)

Unpacks a ZIP file and moves contents with platform filtering.

Parameters:

Name	Type	Description	Default
zip_path	str	Path to ZIP file	required
base_path	str	Base installation path	'./mods'
module_name	str	Module name (extracted from ZIP if not provided)	''
target_platform	Optional[Platform]	Optional platform filter for installation	None

Returns:

Type	Description
Optional[str]	Name of installed module or None on failure

Source code in toolboxv2/mods/CloudM/ModManager.py

def unpack_and_move_module(zip_path: str, base_path: str = './mods',
                           module_name: str = '',
                           target_platform: Optional[Platform] = None) -> Optional[str]:
    """
    Unpacks a ZIP file and moves contents with platform filtering.

    Args:
        zip_path: Path to ZIP file
        base_path: Base installation path
        module_name: Module name (extracted from ZIP if not provided)
        target_platform: Optional platform filter for installation

    Returns:
        Name of installed module or None on failure
    """
    zip_path = Path(zip_path)
    base_path = Path(base_path)

    if not module_name:
        module_name = zip_path.name.split('$')[1].split('&')[0]

    module_path = base_path / module_name
    temp_base = Path('./mods_sto/temp')

    try:
        temp_base.mkdir(parents=True, exist_ok=True)

        with tempfile.TemporaryDirectory(dir=str(temp_base)) as temp_dir:
            temp_dir = Path(temp_dir)

            with Spinner(f"Extracting {zip_path.name}"):
                with zipfile.ZipFile(zip_path, 'r') as zip_ref:
                    zip_ref.extractall(temp_dir)

            # Load configuration to check for platform-specific installation
            config_path = temp_dir / module_name / "tbConfig.yaml"
            if not config_path.exists():
                config_path = temp_dir / f"{module_name}.yaml"

            config, errors = load_and_validate_config(config_path)

            if errors:
                print(f"⚠ Configuration validation warnings: {', '.join(errors)}")

            # Handle module directory
            source_module = temp_dir / module_name

            if source_module.exists():
                with Spinner(f"Installing module to {module_path}"):
                    if module_path.exists():
                        shutil.rmtree(module_path)

                    # If platform filtering is enabled and config exists
                    if target_platform and config:
                        platform_files = get_platform_files(config, target_platform)

                        # Install only platform-specific files
                        module_path.mkdir(parents=True, exist_ok=True)

                        for pattern in platform_files:
                            if pattern == "*":
                                # Copy all files
                                shutil.copytree(source_module, module_path, dirs_exist_ok=True)
                                break
                            else:
                                # Copy specific files/patterns
                                for file in source_module.glob(pattern):
                                    if file.is_file():
                                        shutil.copy2(file, module_path)
                                    elif file.is_dir():
                                        shutil.copytree(file, module_path / file.name, dirs_exist_ok=True)
                    else:
                        # Install all files
                        shutil.copytree(source_module, module_path, dirs_exist_ok=True)

            # Handle additional files in root
            with Spinner("Installing additional files"):
                for item in temp_dir.iterdir():
                    if item.name == module_name or item.name.endswith('.yaml'):
                        continue

                    target = Path('./') / item.name
                    if item.is_dir():
                        if target.exists():
                            shutil.rmtree(target)
                        shutil.copytree(item, target, dirs_exist_ok=True)
                    else:
                        shutil.copy2(item, target)

            print(f"✓ Successfully installed/updated module: {module_name}")
            return module_name

    except Exception as e:
        print(f"✗ Error during installation: {str(e)}")
        if module_path.exists():
            shutil.rmtree(module_path)
        raise

update_all_mods(app) async

Updates all installed modules.

Parameters:

Name	Type	Description	Default
app	Optional[App]	Application instance	required

Returns:

Type	Description
Result	Result with update summary

Source code in toolboxv2/mods/CloudM/ModManager.py

@export(mod_name=Name, name="update_all", test=False)
async def update_all_mods(app: Optional[App]) -> Result:
    """
    Updates all installed modules.

    Args:
        app: Application instance

    Returns:
        Result with update summary
    """
    if app is None:
        app = get_app(f"{Name}.update_all")

    all_mods = app.get_all_mods()
    results = {"updated": [], "failed": [], "up_to_date": []}

    async def check_and_update(mod_name: str):
        try:
            # Get remote version
            response = await app.session.fetch(
                f"/api/{Name}/getModVersion?module_name={mod_name}"
            )
            remote_version = await response.text()
            remote_version = remote_version.strip('"') if remote_version != "None" else None

            if not remote_version:
                results["failed"].append({"module": mod_name, "reason": "Version not found"})
                return

            local_mod = app.get_mod(mod_name)
            if not local_mod:
                results["failed"].append({"module": mod_name, "reason": "Local module not found"})
                return

            local_version = getattr(local_mod, 'version', '0.0.0')

            if pv.parse(remote_version) > pv.parse(local_version):
                result = await installer(app, mod_name, build_state=False)
                if result.is_error:
                    results["failed"].append({"module": mod_name, "reason": str(result)})
                else:
                    results["updated"].append({"module": mod_name, "version": remote_version})
            else:
                results["up_to_date"].append(mod_name)

        except Exception as e:
            results["failed"].append({"module": mod_name, "reason": str(e)})

    # Run updates in parallel
    await asyncio.gather(*[check_and_update(mod) for mod in all_mods])

    # Rebuild state once at the end
    with Spinner("Rebuilding application state"):
        get_state_from_app(app)

    return Result.ok({
        "summary": {
            "total": len(all_mods),
            "updated": len(results["updated"]),
            "up_to_date": len(results["up_to_date"]),
            "failed": len(results["failed"])
        },
        "details": results
    })

upload(app, module_name) async

Uploads an existing module package to the server.

Parameters:

Name	Type	Description	Default
app	Optional[App]	Application instance	required
module_name	str	Name of module to upload	required

Returns:

Type	Description
Result	Result with upload status

Source code in toolboxv2/mods/CloudM/ModManager.py

@export(mod_name=Name, name="upload", test=False)
async def upload(app: Optional[App], module_name: str) -> Result:
    """
    Uploads an existing module package to the server.

    Args:
        app: Application instance
        module_name: Name of module to upload

    Returns:
        Result with upload status
    """
    if app is None:
        app = get_app(f"{Name}.upload")

    try:
        zip_path = find_highest_zip_version(module_name)

        if not zip_path:
            return Result.default_user_error(f"No package found for module '{module_name}'")

        confirm = input(f"Upload ZIP file {zip_path}? (y/n): ")
        if 'y' not in confirm.lower():
            return Result.ok("Upload cancelled")

        res = await app.session.upload_file(zip_path, f'/api/{Name}/upload_mod')

        return Result.ok({
            "message": "Upload completed",
            "response": res
        })

    except Exception as e:
        return Result.default_internal_error(f"Upload failed: {str(e)}")

upload_mod(app, request, form_data=None) async

Uploads a module ZIP file to the server.

Parameters:

Name	Type	Description	Default
app	App	Application instance	required
request	RequestData	Request data	required
form_data	Optional[Dict[str, Any]]	Form data containing file	None

Returns:

Type	Description
Result	Result with upload status

Source code in toolboxv2/mods/CloudM/ModManager.py

@export(mod_name=Name, name="upload_mod", api=True, api_methods=['POST'])
async def upload_mod(app: App, request: RequestData,
                     form_data: Optional[Dict[str, Any]] = None) -> Result:
    """
    Uploads a module ZIP file to the server.

    Args:
        app: Application instance
        request: Request data
        form_data: Form data containing file

    Returns:
        Result with upload status
    """
    if not isinstance(form_data, dict):
        return Result.default_user_error("No form data provided")

    if form_data is None or 'files' not in form_data:
        return Result.default_user_error("No file provided")

    try:
        uploaded_file = form_data.get('files')[0]
        file_name = uploaded_file.filename
        file_bytes = uploaded_file.file.read()

        # Security validation
        if not file_name.endswith('.zip'):
            return Result.default_user_error("Only ZIP files are allowed")

        if not file_name.startswith('RST$'):
            return Result.default_user_error("Invalid module ZIP format")

        # Save file
        save_path = Path(app.start_dir) / "mods_sto" / file_name
        save_path.parent.mkdir(parents=True, exist_ok=True)
        save_path.write_bytes(file_bytes)

        # Extract module info
        module_name = file_name.split('$')[1].split('&')[0]
        module_version = file_name.split('§')[1].replace('.zip', '')

        return Result.ok({
            "message": f"Successfully uploaded {file_name}",
            "module": module_name,
            "version": module_version,
            "size": len(file_bytes)
        })

    except Exception as e:
        return Result.default_internal_error(f"Upload failed: {str(e)}")

validate_config(config, schema)

Validates configuration against schema.

Parameters:

Name	Type	Description	Default
config	Dict	Configuration dictionary to validate	required
schema	Dict	Schema dictionary with expected types	required

Returns:

Type	Description
Tuple[bool, List[str]]	Tuple of (is_valid, list_of_errors)

Source code in toolboxv2/mods/CloudM/ModManager.py

def validate_config(config: Dict, schema: Dict) -> Tuple[bool, List[str]]:
    """
    Validates configuration against schema.

    Args:
        config: Configuration dictionary to validate
        schema: Schema dictionary with expected types

    Returns:
        Tuple of (is_valid, list_of_errors)
    """
    errors = []

    def check_type(key: str, value: Any, expected_type: Any, path: str = ""):
        full_path = f"{path}.{key}" if path else key

        if isinstance(expected_type, dict):
            if not isinstance(value, dict):
                errors.append(f"{full_path}: Expected dict, got {type(value).__name__}")
                return
            for sub_key, sub_type in expected_type.items():
                if sub_key in value:
                    check_type(sub_key, value[sub_key], sub_type, full_path)
        elif expected_type == list:
            if not isinstance(value, list):
                errors.append(f"{full_path}: Expected list, got {type(value).__name__}")
        elif expected_type == dict:
            if not isinstance(value, dict):
                errors.append(f"{full_path}: Expected dict, got {type(value).__name__}")
        elif not isinstance(value, expected_type):
            errors.append(f"{full_path}: Expected {expected_type.__name__}, got {type(value).__name__}")

    for key, expected_type in schema.items():
        if key in config:
            check_type(key, config[key], expected_type)

    return len(errors) == 0, errors

ModManager_tests

TestModManager

Bases: TestCase

Source code in toolboxv2/mods/CloudM/ModManager_tests.py

class TestModManager(unittest.TestCase):
    app: App = None

    def test_increment_version(self):
        """Tests the version increment logic."""
        print("\nTesting increment_version...")
        self.assertEqual(increment_version("v0.0.1"), "v0.0.2")
        self.assertEqual(increment_version("v0.0.99", max_value=99), "v0.1.0")
        self.assertEqual(increment_version("v0.99.99", max_value=99), "v1.0.0")
        self.assertEqual(increment_version("v98"), "v99")
        with self.assertRaises(ValueError, msg="Should fail if 'v' is missing"):
            print(increment_version("0.0.1"))
        print("increment_version tests passed.")

    def setUp(self):
        """Set up a temporary environment for each test."""
        self.original_cwd = os.getcwd()
        self.test_dir = tempfile.mkdtemp(prefix="mod_manager_test_")

        # The functions in ModManager use relative paths like './mods' and './mods_sto'
        # We'll create these inside our temp directory and chdir into it.
        os.chdir(self.test_dir)
        os.makedirs("mods", exist_ok=True)
        os.makedirs("mods_sto", exist_ok=True)
        os.makedirs("source_module", exist_ok=True)

    def tearDown(self):
        """Clean up the temporary environment after each test."""
        os.chdir(self.original_cwd)
        shutil.rmtree(self.test_dir, ignore_errors=True)

    def test_create_pack_unpack_cycle(self):
        """Tests the full cycle of creating, packing, and unpacking a module."""
        print("\nTesting create_pack_unpack_cycle...")
        module_name = "MyTestMod"
        module_version = "v0.1.0"

        # 1. Create a dummy module structure inside the temp 'source_module' dir
        source_path = Path("source_module")
        module_source_path = source_path / module_name
        module_source_path.mkdir()
        (module_source_path / "main.py").write_text("print('hello from my test mod')")
        (module_source_path / "data.txt").write_text("some test data")

        # 2. Call create_and_pack_module
        # The 'path' argument is the parent directory of the module directory.
        zip_path_str = create_and_pack_module(
            path=str(source_path),
            module_name=module_name,
            version=module_version
        )
        self.assertTrue(zip_path_str, "create_and_pack_module should return a path.")
        zip_path = Path(zip_path_str)

        # 3. Assert the zip file was created in the correct location ('./mods_sto')
        self.assertTrue(zip_path.exists(), f"Zip file should exist at {zip_path}")
        self.assertEqual(zip_path.parent.name, "mods_sto")

        # 4. Call unpack_and_move_module
        # We unpack into the './mods' directory.
        unpacked_name = unpack_and_move_module(
            zip_path=str(zip_path),
            base_path="mods"
        )

        # 5. Assert the module was unpacked correctly
        self.assertEqual(unpacked_name, module_name)
        unpacked_dir = Path("mods") / module_name
        self.assertTrue(unpacked_dir.is_dir(), "Unpacked module directory should exist.")

        # Verify content
        self.assertTrue((unpacked_dir / "main.py").exists())
        self.assertEqual((unpacked_dir / "main.py").read_text(), "print('hello from my test mod')")
        self.assertTrue((unpacked_dir / "data.txt").exists())
        self.assertEqual((unpacked_dir / "data.txt").read_text(), "some test data")

        # Verify that the tbConfig.yaml was created and has correct info
        config_path = unpacked_dir / "tbConfig.yaml"
        self.assertTrue(config_path.exists())
        with open(config_path) as f:
            config = yaml.safe_load(f)
        self.assertEqual(config.get("module_name"), module_name)
        self.assertEqual(config.get("version"), module_version)

        print("create_pack_unpack_cycle tests passed.")

    def test_install_from_zip(self):
        """Tests the install_from_zip helper function."""
        print("\nTesting install_from_zip...")
        module_name = "MyInstallTestMod"
        module_version = "v0.1.1"

        # 1. Create a dummy module and zip it
        source_path = Path("source_module")
        module_source_path = source_path / module_name
        module_source_path.mkdir()
        (module_source_path / "main.py").write_text("pass")
        zip_path_str = create_and_pack_module(
            path=str(source_path),
            module_name=module_name,
            version=module_version
        )
        zip_path = Path(zip_path_str)
        zip_name = zip_path.name

        # 2. Mock the app object needed by install_from_zip
        mock_app = lambda :None
        mock_app.start_dir = self.test_dir

        # 3. Call install_from_zip
        result = install_from_zip(mock_app, zip_name, no_dep=True)

        # 4. Assert the installation was successful
        self.assertTrue(result)
        unpacked_dir = Path("mods") / module_name
        self.assertTrue(unpacked_dir.is_dir())
        self.assertTrue((unpacked_dir / "main.py").exists())
        print("install_from_zip tests passed.")

setUp()

Set up a temporary environment for each test.

Source code in toolboxv2/mods/CloudM/ModManager_tests.py

def setUp(self):
    """Set up a temporary environment for each test."""
    self.original_cwd = os.getcwd()
    self.test_dir = tempfile.mkdtemp(prefix="mod_manager_test_")

    # The functions in ModManager use relative paths like './mods' and './mods_sto'
    # We'll create these inside our temp directory and chdir into it.
    os.chdir(self.test_dir)
    os.makedirs("mods", exist_ok=True)
    os.makedirs("mods_sto", exist_ok=True)
    os.makedirs("source_module", exist_ok=True)

tearDown()

Clean up the temporary environment after each test.

Source code in toolboxv2/mods/CloudM/ModManager_tests.py

def tearDown(self):
    """Clean up the temporary environment after each test."""
    os.chdir(self.original_cwd)
    shutil.rmtree(self.test_dir, ignore_errors=True)

test_create_pack_unpack_cycle()

Tests the full cycle of creating, packing, and unpacking a module.

Source code in toolboxv2/mods/CloudM/ModManager_tests.py

def test_create_pack_unpack_cycle(self):
    """Tests the full cycle of creating, packing, and unpacking a module."""
    print("\nTesting create_pack_unpack_cycle...")
    module_name = "MyTestMod"
    module_version = "v0.1.0"

    # 1. Create a dummy module structure inside the temp 'source_module' dir
    source_path = Path("source_module")
    module_source_path = source_path / module_name
    module_source_path.mkdir()
    (module_source_path / "main.py").write_text("print('hello from my test mod')")
    (module_source_path / "data.txt").write_text("some test data")

    # 2. Call create_and_pack_module
    # The 'path' argument is the parent directory of the module directory.
    zip_path_str = create_and_pack_module(
        path=str(source_path),
        module_name=module_name,
        version=module_version
    )
    self.assertTrue(zip_path_str, "create_and_pack_module should return a path.")
    zip_path = Path(zip_path_str)

    # 3. Assert the zip file was created in the correct location ('./mods_sto')
    self.assertTrue(zip_path.exists(), f"Zip file should exist at {zip_path}")
    self.assertEqual(zip_path.parent.name, "mods_sto")

    # 4. Call unpack_and_move_module
    # We unpack into the './mods' directory.
    unpacked_name = unpack_and_move_module(
        zip_path=str(zip_path),
        base_path="mods"
    )

    # 5. Assert the module was unpacked correctly
    self.assertEqual(unpacked_name, module_name)
    unpacked_dir = Path("mods") / module_name
    self.assertTrue(unpacked_dir.is_dir(), "Unpacked module directory should exist.")

    # Verify content
    self.assertTrue((unpacked_dir / "main.py").exists())
    self.assertEqual((unpacked_dir / "main.py").read_text(), "print('hello from my test mod')")
    self.assertTrue((unpacked_dir / "data.txt").exists())
    self.assertEqual((unpacked_dir / "data.txt").read_text(), "some test data")

    # Verify that the tbConfig.yaml was created and has correct info
    config_path = unpacked_dir / "tbConfig.yaml"
    self.assertTrue(config_path.exists())
    with open(config_path) as f:
        config = yaml.safe_load(f)
    self.assertEqual(config.get("module_name"), module_name)
    self.assertEqual(config.get("version"), module_version)

    print("create_pack_unpack_cycle tests passed.")

test_increment_version()

Tests the version increment logic.

Source code in toolboxv2/mods/CloudM/ModManager_tests.py

def test_increment_version(self):
    """Tests the version increment logic."""
    print("\nTesting increment_version...")
    self.assertEqual(increment_version("v0.0.1"), "v0.0.2")
    self.assertEqual(increment_version("v0.0.99", max_value=99), "v0.1.0")
    self.assertEqual(increment_version("v0.99.99", max_value=99), "v1.0.0")
    self.assertEqual(increment_version("v98"), "v99")
    with self.assertRaises(ValueError, msg="Should fail if 'v' is missing"):
        print(increment_version("0.0.1"))
    print("increment_version tests passed.")

test_install_from_zip()

Tests the install_from_zip helper function.

Source code in toolboxv2/mods/CloudM/ModManager_tests.py

def test_install_from_zip(self):
    """Tests the install_from_zip helper function."""
    print("\nTesting install_from_zip...")
    module_name = "MyInstallTestMod"
    module_version = "v0.1.1"

    # 1. Create a dummy module and zip it
    source_path = Path("source_module")
    module_source_path = source_path / module_name
    module_source_path.mkdir()
    (module_source_path / "main.py").write_text("pass")
    zip_path_str = create_and_pack_module(
        path=str(source_path),
        module_name=module_name,
        version=module_version
    )
    zip_path = Path(zip_path_str)
    zip_name = zip_path.name

    # 2. Mock the app object needed by install_from_zip
    mock_app = lambda :None
    mock_app.start_dir = self.test_dir

    # 3. Call install_from_zip
    result = install_from_zip(mock_app, zip_name, no_dep=True)

    # 4. Assert the installation was successful
    self.assertTrue(result)
    unpacked_dir = Path("mods") / module_name
    self.assertTrue(unpacked_dir.is_dir())
    self.assertTrue((unpacked_dir / "main.py").exists())
    print("install_from_zip tests passed.")

run_mod_manager_tests(app)

This function will be automatically discovered and run by the test runner. It uses the standard unittest framework to run tests.

Source code in toolboxv2/mods/CloudM/ModManager_tests.py

@export(test_only=True)
def run_mod_manager_tests(app: App):
    """
    This function will be automatically discovered and run by the test runner.
    It uses the standard unittest framework to run tests.
    """
    print("Running ModManager Tests...")
    # We pass the app instance to the test class so it can be used if needed.
    TestModManager.app = app
    suite = unittest.TestSuite()
    suite.addTest(unittest.makeSuite(TestModManager))
    runner = unittest.TextTestRunner()
    result = runner.run(suite)
    if not result.wasSuccessful():
        # Raise an exception to signal failure to the toolboxv2 test runner
        raise AssertionError(f"ModManager tests failed: {result.errors} {result.failures}")
    print("ModManager tests passed successfully.")
    return True

UserAccountManager

get_current_user_from_request_api_wrapper(app, request) async

API callable version of get_current_user_from_request for tbjs/admin panel

Source code in toolboxv2/mods/CloudM/UserAccountManager.py

@export(mod_name=Name, api=True, version=version, request_as_kwarg=True, row=False)  # row=False to return JSON
async def get_current_user_from_request_api_wrapper(app: App, request: RequestData):
    """ API callable version of get_current_user_from_request for tbjs/admin panel """
    user = await get_current_user_from_request(app, request)
    if not user:
        # Return error that tbjs can handle
        return Result.default_user_error(info="User not authenticated or found.", data=None, exec_code=401)
    user_dict = asdict(user)
    pub_user_data = {}
    for key in ['name','pub_key','email','creation_time','is_persona','level','log_level','settings']:
        pub_user_data[key] = user_dict.get(key, None)
    return Result.ok(data=pub_user_data)

UserDashboard

close_cli_session(app, request, cli_session_id) async

Close a specific CLI session

Source code in toolboxv2/mods/CloudM/UserDashboard.py

@export(mod_name=Name, version=version, api=True, request_as_kwarg=True)
async def close_cli_session(app: App, request: RequestData, cli_session_id: str):
    """Close a specific CLI session"""
    current_user = await get_current_user_from_request(app, request)
    if not current_user:
        return Result.default_user_error(info="User not authenticated.", exec_code=401)

    from .UserInstances import close_cli_session as close_cli_session_internal, UserInstances

    # Verify session belongs to current user
    if cli_session_id in UserInstances().cli_sessions:
        session_data = UserInstances().cli_sessions[cli_session_id]
        if session_data['uid'] != current_user.uid:
            return Result.default_user_error(info="Unauthorized to close this session.")

    result = close_cli_session_internal(cli_session_id)
    return Result.ok(data_info="CLI session closed", data={"message": result})

configure_jwt_settings(app=None, ttl_hours=24, allowed_mods=None)

Configure JWT settings for CLI sessions

Source code in toolboxv2/mods/CloudM/UserDashboard.py

@export(mod_name=Name, version=version, api=True)
def configure_jwt_settings(app: App = None, ttl_hours: int = 24, allowed_mods: list = None):
    """Configure JWT settings for CLI sessions"""
    if app is None:
        app = get_app("CloudM.configure_jwt_settings")

    if allowed_mods is None:
        allowed_mods = ["CloudM", "DB", "FileHandler"]

    jwt_config = {
        'ttl_hours': ttl_hours,
        'allowed_mods': allowed_mods,
        'secret_key': app.config_fh.get_file_handler("jwt_secret") or _generate_jwt_secret(app),
        'algorithm': 'HS256'
    }

    app.config_fh.add_to_save_file_handler("jwt_config", jwt_config)
    return Result.ok(data_info="JWT configuration updated", data=jwt_config)

generate_cli_jwt(app=None, username=None, custom_ttl=None)

Generate JWT token for CLI access

Source code in toolboxv2/mods/CloudM/UserDashboard.py

@export(mod_name=Name, version=version, api=True)
def generate_cli_jwt(app: App = None, username: str = None, custom_ttl: int = None):
    """Generate JWT token for CLI access"""
    if app is None:
        app = get_app("CloudM.generate_cli_jwt")

    if not username:
        return Result.default_user_error("Username required")

    jwt_config = app.config_fh.get_file_handler("jwt_config")
    if not jwt_config:
        # Use default configuration
        configure_jwt_settings(app)
        jwt_config = app.config_fh.get_file_handler("jwt_config")

    ttl_hours = custom_ttl or jwt_config.get('ttl_hours', 24)

    payload = {
        'username': username,
        'iat': int(time.time()),
        'exp': int(time.time()) + (ttl_hours * 3600),
        'type': 'cli_access',
        'allowed_mods': jwt_config.get('allowed_mods', [])
    }

    token = jwt.encode(
        payload,
        jwt_config['secret_key'],
        algorithm=jwt_config['algorithm']
    )

    return Result.ok(data_info="JWT generated", data={'token': token, 'expires_in': ttl_hours * 3600})

get_jwt_dashboard(app=None)

Get JWT management dashboard data

Source code in toolboxv2/mods/CloudM/UserDashboard.py

@export(mod_name=Name, version=version, api=True)
def get_jwt_dashboard(app: App = None):
    """Get JWT management dashboard data"""
    if app is None:
        app = get_app("CloudM.get_jwt_dashboard")

    jwt_config = app.config_fh.get_file_handler("jwt_config")

    dashboard_data = {
        'jwt_configured': jwt_config is not None,
        'current_config': jwt_config or {},
        'active_sessions': _get_active_sessions(app),
        'default_ttl': jwt_config.get('ttl_hours', 24) if jwt_config else 24
    }

    return Result.ok(data_info="Dashboard data", data=dashboard_data)

get_my_active_instances_with_cli(app, request) async

Get active instances including CLI sessions

Source code in toolboxv2/mods/CloudM/UserDashboard.py

@export(mod_name=Name, version=version, api=True, request_as_kwarg=True)
async def get_my_active_instances_with_cli(app: App, request: RequestData):
    """Get active instances including CLI sessions"""
    current_user = await get_current_user_from_request(app, request)
    if not current_user:
        return Result.default_user_error(info="User not authenticated.", exec_code=401)

    # Get enhanced instance data with CLI sessions
    from .UserInstances import get_user_instance_with_cli_sessions, get_user_cli_sessions

    instance_data_result = get_user_instance_with_cli_sessions(current_user.uid, hydrate=True)
    cli_sessions = get_user_cli_sessions(current_user.uid)

    active_instances_output = []
    if instance_data_result and isinstance(instance_data_result, dict):
        live_modules_info = []
        if instance_data_result.get("live"):
            for mod_name, spec_val in instance_data_result.get("live").items():
                live_modules_info.append({"name": mod_name, "spec": str(spec_val)})

        instance_summary = {
            "SiID": instance_data_result.get("SiID"),
            "VtID": instance_data_result.get("VtID"),
            "webSocketID": instance_data_result.get("webSocketID"),
            "live_modules": live_modules_info,
            "saved_modules": instance_data_result.get("save", {}).get("mods", []),
            "cli_sessions": cli_sessions,
            "active_cli_sessions": len([s for s in cli_sessions if s.get('status') == 'active'])
        }
        active_instances_output.append(instance_summary)

    return Result.ok(data_info="Active instances with CLI sessions retrieved", data=active_instances_output)

validate_cli_jwt(app=None, token=None)

Validate CLI JWT token

Source code in toolboxv2/mods/CloudM/UserDashboard.py

@export(mod_name=Name, version=version, api=True)
def validate_cli_jwt(app: App = None, token: str = None):
    """Validate CLI JWT token"""
    if app is None:
        app = get_app("CloudM.validate_cli_jwt")

    if not token:
        return Result.default_user_error("Token required")

    jwt_config = app.config_fh.get_file_handler("jwt_config")
    if not jwt_config:
        return Result.default_internal_error("JWT configuration not found")

    try:
        payload = jwt.decode(
            token,
            jwt_config['secret_key'],
            algorithms=[jwt_config['algorithm']]
        )

        # Check if token is for CLI access
        if payload.get('type') != 'cli_access':
            return Result.default_user_error("Invalid token type")

        return Result.ok(data_info="Token valid", data=payload)

    except jwt.ExpiredSignatureError:
        return Result.default_user_error("Token expired")
    except jwt.InvalidTokenError:
        return Result.default_user_error("Invalid token")

UserInstances

cleanup_expired_cli_sessions(max_age_hours=24)

Clean up expired CLI sessions

Source code in toolboxv2/mods/CloudM/UserInstances.py

@e
def cleanup_expired_cli_sessions(max_age_hours: int = 24):
    """Clean up expired CLI sessions"""
    current_time = time.time()
    max_age_seconds = max_age_hours * 3600

    expired_sessions = []
    for session_id, session_data in list(UserInstances().cli_sessions.items()):
        if current_time - session_data['last_activity'] > max_age_seconds:
            expired_sessions.append(session_id)

    for session_id in expired_sessions:
        close_cli_session(session_id)

    logger.info(f"Cleaned up {len(expired_sessions)} expired CLI sessions")
    return f"Cleaned up {len(expired_sessions)} expired CLI sessions"

close_cli_session(cli_session_id)

Close a CLI session

Source code in toolboxv2/mods/CloudM/UserInstances.py

@e
def close_cli_session(cli_session_id: str):
    """Close a CLI session"""
    if cli_session_id not in UserInstances().cli_sessions:
        return "CLI session not found"

    session_data = UserInstances().cli_sessions[cli_session_id]
    session_data['status'] = 'closed'
    session_data['closed_at'] = time.time()

    # Remove from active sessions
    del UserInstances().cli_sessions[cli_session_id]

    # Update persistent storage to mark as closed
    app.run_any('DB', 'set',
                query=f"CLI::Session::{session_data['uid']}::{cli_session_id}",
                data=json.dumps(session_data))

    logger.info(f"CLI session {cli_session_id} closed")
    return "CLI session closed successfully"

get_all_active_cli_sessions()

Get all active CLI sessions

Source code in toolboxv2/mods/CloudM/UserInstances.py

@e
def get_all_active_cli_sessions():
    """Get all active CLI sessions"""
    return list(UserInstances().cli_sessions.values())

get_instance_overview(si_id=None)

Get comprehensive overview of user instances and CLI sessions

Source code in toolboxv2/mods/CloudM/UserInstances.py

@e
def get_instance_overview(si_id: str = None):
    """Get comprehensive overview of user instances and CLI sessions"""
    overview = {
        'web_instances': {},
        'cli_sessions': {},
        'total_active_web': 0,
        'total_active_cli': 0
    }

    # Web instances
    if si_id:
        if si_id in UserInstances().live_user_instances:
            overview['web_instances'][si_id] = UserInstances().live_user_instances[si_id]
            overview['total_active_web'] = 1
    else:
        overview['web_instances'] = UserInstances().live_user_instances.copy()
        overview['total_active_web'] = len(UserInstances().live_user_instances)

    # CLI sessions
    overview['cli_sessions'] = UserInstances().cli_sessions.copy()
    overview['total_active_cli'] = len(UserInstances().cli_sessions)

    return overview

get_user_cli_sessions(uid)

Get all CLI sessions for a user

Source code in toolboxv2/mods/CloudM/UserInstances.py

@e
def get_user_cli_sessions(uid: str):
    """Get all CLI sessions for a user"""
    if uid is None:
        return []

    # Get active sessions from memory
    active_sessions = []
    for session_id, session_data in UserInstances().cli_sessions.items():
        if session_data['uid'] == uid:
            active_sessions.append(session_data)

    # Also check persistent storage for recent sessions
    try:
        # This would need a query pattern to get all CLI sessions for a user
        # For now, return active sessions
        pass
    except Exception as e:
        logger.warning(f"Error fetching persistent CLI sessions: {e}")

    return active_sessions

get_user_instance_with_cli_sessions(uid, hydrate=True)

Enhanced get_user_instance that includes CLI sessions

Source code in toolboxv2/mods/CloudM/UserInstances.py

@e
def get_user_instance_with_cli_sessions(uid: str, hydrate: bool = True):
    """Enhanced get_user_instance that includes CLI sessions"""
    # Get regular user instance
    instance = get_user_instance(uid, hydrate)

    if instance:
        # Add CLI sessions information
        cli_sessions = get_user_cli_sessions(uid)
        instance['cli_sessions'] = cli_sessions
        instance['active_cli_sessions'] = len([s for s in cli_sessions if s['status'] == 'active'])

    return instance

register_cli_session(uid, jwt_token, session_info=None)

Register a new CLI session

Source code in toolboxv2/mods/CloudM/UserInstances.py

@e
def register_cli_session(uid: str, jwt_token: str, session_info: dict = None):
    """Register a new CLI session"""
    if uid is None:
        return Result.default_user_error("UID required")

    cli_session_id = UserInstances.get_cli_session_id(uid).get()

    session_data = {
        'uid': uid,
        'cli_session_id': cli_session_id,
        'jwt_token': jwt_token,
        'created_at': time.time(),
        'last_activity': time.time(),
        'status': 'active',
        'session_info': session_info or {}
    }

    UserInstances().cli_sessions[cli_session_id] = session_data

    # Save to persistent storage
    app.run_any('DB', 'set',
                query=f"CLI::Session::{uid}::{cli_session_id}",
                data=json.dumps(session_data))

    logger.info(f"CLI session registered for user {uid}")
    return Result.ok("CLI session registered", data=session_data)

update_cli_session_activity(cli_session_id)

Update last activity timestamp for CLI session

Source code in toolboxv2/mods/CloudM/UserInstances.py

@e
def update_cli_session_activity(cli_session_id: str):
    """Update last activity timestamp for CLI session"""
    if cli_session_id in UserInstances().cli_sessions:
        UserInstances().cli_sessions[cli_session_id]['last_activity'] = time.time()
        session_data = UserInstances().cli_sessions[cli_session_id]

        # Update persistent storage
        app.run_any('DB', 'set',
                    query=f"CLI::Session::{session_data['uid']}::{cli_session_id}",
                    data=json.dumps(session_data))
        return True
    return False

email_services

send_email_verification_email(app, user_email, username, verification_url)

Sends an email verification link to the user.

Source code in toolboxv2/mods/CloudM/email_services.py

@s_export
def send_email_verification_email(app: App, user_email: str, username: str, verification_url: str):
    """Sends an email verification link to the user."""
    sender = EmailSender(app)
    subject = f"Verify Your Email for {APP_NAME}"
    preview_text = f"Almost there, {username}! Just one more step to activate your account."

    content_html = f"""
        <h2>Hi {username},</h2>
        <p>Thanks for signing up for {APP_NAME}! To complete your registration, please verify your email address by clicking the button below.</p>
        <a href="{verification_url}" class="button">Verify Email Address</a>
        <p>If you didn't create an account with {APP_NAME}, you can safely ignore this email.</p>
        <p>If the button doesn't work, copy and paste this link into your browser:<br><span class="link-in-text">{verification_url}</span></p>
        <p>Sincerely,<br>The {APP_NAME} Team</p>
    """
    return sender.send_html_email(user_email, subject, content_html, preview_text)

send_magic_link_email(app, user_email, magic_link_url, username=None)

Sends a magic link email for login.

Source code in toolboxv2/mods/CloudM/email_services.py

@s_export
def send_magic_link_email(app: App, user_email: str, magic_link_url: str, username: str = None):
    """Sends a magic link email for login."""
    sender = EmailSender(app)
    greeting_name = f", {username}" if username else ""
    subject = f"Your Magic Login Link for {APP_NAME}"
    preview_text = "Securely access your account with this one-time link."

    content_html = f"""
        <h2>Hello{greeting_name}!</h2>
        <p>You requested a magic link to sign in to your {APP_NAME} account.</p>
        <p>Click the button below to log in. This link is temporary and will expire shortly.</p>
        <a href="{magic_link_url}" class="button">Log In Securely</a>
        <p> Invitation key: {magic_link_url.split('?key=')[1].split('&name=')[0].replace('%23', '#')}</p>
        <p>If you did not request this link, please ignore this email. Your account is safe.</p>
        <p>If the button doesn't work, copy and paste this link into your browser:<br><span class="link-in-text">{magic_link_url}</span></p>
        <p>Thanks,<br>The {APP_NAME} Team</p>
    """
    return sender.send_html_email(user_email, subject, content_html, preview_text)

send_signup_invitation_email(app, invited_user_email, invited_username, inviter_username=None)

Generates an invitation link and sends it via email.

Source code in toolboxv2/mods/CloudM/email_services.py

@s_export
def send_signup_invitation_email(app: App, invited_user_email: str, invited_username: str,
                                 inviter_username: str = None):
    """Generates an invitation link and sends it via email."""
    sender = EmailSender(app)

    # Generate invitation code as specified in the prompt
    # This uses the Code class, assuming TB_R_KEY is set in the environment
    invitation_code = Code.one_way_hash(invited_username, "00#", os.getenv("TB_R_KEY", "pepper123"))[:12] + str(
        uuid.uuid4())[:6]

    # Construct the signup link URL (adjust your frontend signup path as needed)
    signup_link_url = f"{APP_BASE_URL}/web/assets/signup.html?invitation={quote(invitation_code)}&email={quote(invited_user_email)}&username={quote(invited_username)}"

    subject = f"You're Invited to Join {APP_NAME}!"
    preview_text = f"{inviter_username or 'A friend'} has invited you to {APP_NAME}!"
    inviter_line = f"<p>{inviter_username} has invited you to join.</p>" if inviter_username else "<p>You've been invited to join.</p>"

    content_html = f"""
        <h2>Hello {invited_username},</h2>
        {inviter_line}
        <p>{APP_NAME} is an exciting platform, and we'd love for you to be a part of it!</p>
        <p>Click the button below to accept the invitation and create your account:</p>
        <a href="{signup_link_url}" class="button">Accept Invitation & Sign Up</a>
        <p>This invitation is unique to you : {invitation_code}</p>
        <p>If the button doesn't work, copy and paste this link into your browser:<br><span class="link-in-text">{signup_link_url}</span></p>
        <p>We look forward to seeing you there!<br>The {APP_NAME} Team</p>
    """
    return sender.send_html_email(invited_user_email, subject, content_html, preview_text)

send_waiting_list_confirmation_email(app, user_email)

Sends a confirmation email for joining the waiting list.

Source code in toolboxv2/mods/CloudM/email_services.py

@s_export
def send_waiting_list_confirmation_email(app: App, user_email: str):
    """Sends a confirmation email for joining the waiting list."""
    sender = EmailSender(app)
    subject = f"You're on the Waiting List for {APP_NAME}!"
    preview_text = "Thanks for your interest! We'll keep you updated."

    content_html = f"""
        <h2>You're In!</h2>
        <p>Thank you for joining the waiting list for {APP_NAME}. We're working hard to get things ready and appreciate your interest.</p>
        <p>We'll notify you as soon as we have updates or when access becomes available.</p>
        <p>In the meantime, you can follow our progress or learn more at <a href="{APP_BASE_URL}" class="link-in-text">{APP_BASE_URL}</a>.</p>
        <p>Stay tuned,<br>The {APP_NAME} Team</p>
    """
    return sender.send_html_email(user_email, subject, content_html, preview_text,
                                  recipient_email_for_unsubscribe=user_email, show_unsubscribe_link=True)

send_welcome_email(app, user_email, username, welcome_action_url=None)

Sends a welcome email to a new user.

Source code in toolboxv2/mods/CloudM/email_services.py

@s_export  # Changed to native, api=False as it's a backend function
def send_welcome_email(app: App, user_email: str, username: str, welcome_action_url: str = None):
    """Sends a welcome email to a new user."""
    sender = EmailSender(app)
    subject = f"Welcome to {APP_NAME}, {username}!"
    preview_text = f"We're thrilled to have you, {username}!"
    action_url = welcome_action_url or f"{APP_BASE_URL}/dashboard"  # Default to dashboard

    content_html = f"""
        <h2>Welcome Aboard, {username}!</h2>
        <p>Thank you for signing up for {APP_NAME}. We're excited to have you join our community!</p>
        <p>Here are a few things you might want to do next:</p>
        <ul>
            <li>Explore your new account features.</li>
            <li>Customize your profile.</li>
        </ul>
        <p>Click the button below to get started:</p>
        <a href="{action_url}" class="button">Go to Your Dashboard</a>
        <p>If the button doesn't work, copy and paste this link into your browser:<br><span class="link-in-text">{action_url}</span></p>
        <p>Best regards,<br>The {APP_NAME} Team</p>
    """
    return sender.send_html_email(user_email, subject, content_html, preview_text,
                                  recipient_email_for_unsubscribe=user_email, show_unsubscribe_link=True)

extras

cli_logout(app=None) async

CLI logout - imports from LogInSystem

Source code in toolboxv2/mods/CloudM/extras.py

@export(mod_name=Name, version=version, state=False)
async def cli_logout(app: App = None):
    """CLI logout - imports from LogInSystem"""
    from .LogInSystem import cli_logout as _cli_logout
    return await _cli_logout(app)

cli_web_login(app=None, force_remote=False, force_local=False) async

Enhanced CLI web login - imports from LogInSystem

Source code in toolboxv2/mods/CloudM/extras.py

@export(mod_name=Name, version=version, state=False)
async def cli_web_login(app: App = None, force_remote: bool = False, force_local: bool = False):
    """Enhanced CLI web login - imports from LogInSystem"""
    from .LogInSystem import cli_web_login as _cli_web_login
    return await _cli_web_login(app, force_remote, force_local)

mini

check_multiple_processes(pids)

Checks the status of multiple processes in a single system call. Returns a dictionary mapping PIDs to their status (GREEN_CIRCLE, RED_CIRCLE, or YELLOW_CIRCLE).

Source code in toolboxv2/mods/CloudM/mini.py

def check_multiple_processes(pids: list[int]) -> dict[int, str]:
    """
    Checks the status of multiple processes in a single system call.
    Returns a dictionary mapping PIDs to their status (GREEN_CIRCLE, RED_CIRCLE, or YELLOW_CIRCLE).
    """
    if not pids:
        return {}

    pid_status = {}

    if os.name == 'nt':  # Windows
        try:
            # Windows tasklist requires separate /FI for each filter
            command = 'tasklist'

            # Add encoding handling for Windows
            result = subprocess.run(
                command,
                capture_output=True,
                text=True,
                shell=True,
                encoding='cp850'  # Use cp850 for Windows console output
            )
            # Create a set of running PIDs from the output
            running_pids = set()
            for line in result.stdout.lower().split('\n'):
                for pid in pids:
                    if str(pid) in line:
                        running_pids.add(pid)
            # Assign status based on whether PID was found in output
            for pid in pids:
                if pid in running_pids:
                    pid_status[pid] = GREEN_CIRCLE
                else:
                    pid_status[pid] = RED_CIRCLE

        except subprocess.SubprocessError as e:
            print(f"SubprocessError: {e}")  # For debugging
            # Mark all as YELLOW_CIRCLE if there's an error running the command
            for pid in pids:
                pid_status[pid] = YELLOW_CIRCLE
        except UnicodeDecodeError as e:
            print(f"UnicodeDecodeError: {e}")  # For debugging
            # Try alternate encoding if cp850 fails
            try:
                result = subprocess.run(
                    command,
                    capture_output=True,
                    text=True,
                    shell=True,
                    encoding='utf-8'
                )
                running_pids = set()
                for line in result.stdout.lower().split('\n'):
                    for pid in pids:
                        if str(pid) in line:
                            running_pids.add(pid)

                for pid in pids:
                    pid_status[pid] = GREEN_CIRCLE if pid in running_pids else RED_CIRCLE
            except Exception as e:
                print(f"Failed with alternate encoding: {e}")  # For debugging
                for pid in pids:
                    pid_status[pid] = YELLOW_CIRCLE

    else:  # Unix/Linux/Mac
        try:
            pids_str = ','.join(str(pid) for pid in pids)
            command = f'ps -p {pids_str} -o pid='

            result = subprocess.run(
                command,
                capture_output=True,
                text=True,
                shell=True,
                encoding='utf-8'
            )
            running_pids = set(int(pid) for pid in result.stdout.strip().split())

            for pid in pids:
                pid_status[pid] = GREEN_CIRCLE if pid in running_pids else RED_CIRCLE

        except subprocess.SubprocessError as e:
            print(f"SubprocessError: {e}")  # For debugging
            for pid in pids:
                pid_status[pid] = YELLOW_CIRCLE

    return pid_status

get_service_pids(info_dir)

Extracts service names and PIDs from pid files.

Source code in toolboxv2/mods/CloudM/mini.py

def get_service_pids(info_dir):
    """Extracts service names and PIDs from pid files."""
    services = {}
    pid_files = [f for f in os.listdir(info_dir) if re.match(r'(.+)-(.+)\.pid', f)]
    for pid_file in pid_files:
        match = re.match(r'(.+)-(.+)\.pid', pid_file)
        if match:
            services_type, service_name = match.groups()
            # Read the PID from the file
            with open(os.path.join(info_dir, pid_file)) as file:
                pid = file.read().strip()
                # Store the PID using a formatted key
                services[f"{service_name} - {services_type}"] = int(pid)
    return services

get_service_status(dir)

Displays the status of all services.

Source code in toolboxv2/mods/CloudM/mini.py

def get_service_status(dir: str) -> str:
    """Displays the status of all services."""
    if time.time()-services_data_sto_last_update_time[0] > 30:
        services = get_service_pids(dir)
        services_data_sto[0] = services
        services_data_sto_last_update_time[0] = time.time()
    else:
        services = services_data_sto[0]
    if not services:
        return "No services found"

    # Get status for all PIDs in a single call
    pid_statuses = check_multiple_processes(list(services.values()))

    # Build the status string
    res_s = "Service(s):" + ("\n" if len(services) > 1 else ' ')
    for service_name, pid in services.items():
        status = pid_statuses.get(pid, YELLOW_CIRCLE)
        res_s += f"{status} {service_name} (PID: {pid})\n"
    services_data_display[0] = res_s.strip()
    return res_s.rstrip()

module

hash_password(password)

Hash a password for storing.

Source code in toolboxv2/mods/CloudM/module.py

def hash_password(password):
    """Hash a password for storing."""
    salt = hashlib.sha256(os.urandom(60)).hexdigest().encode('ascii')
    pwdhash = hashlib.pbkdf2_hmac('sha512', password.encode('utf-8'), salt,
                                  100000)
    pwdhash = binascii.hexlify(pwdhash)
    return (salt + pwdhash).decode('ascii')

verify_password(stored_password, provided_password)

Verify a stored password against one provided by user

Source code in toolboxv2/mods/CloudM/module.py

def verify_password(stored_password, provided_password):
    """Verify a stored password against one provided by user"""
    salt = stored_password[:64]
    stored_password = stored_password[64:]
    pwdhash = hashlib.pbkdf2_hmac('sha512', provided_password.encode('utf-8'),
                                  salt.encode('ascii'), 100000)
    pwdhash = binascii.hexlify(pwdhash).decode('ascii')
    return pwdhash == stored_password

CodeVerification

VerificationSystem

Source code in toolboxv2/mods/CodeVerification.py

class VerificationSystem:
    def __init__(self, tools_db, scope="main"):
        """
        Initialize VerificationSystem with DB Tools integration

        Args:
            tools_db (Tools): Database tools from toolboxv2.mods.DB
            scope (str, optional): Scope for templates and codes. Defaults to "main".
        """
        self.tools_db = tools_db
        self.scope = scope
        self.tidmp = {}
        self._ensure_scope_templates()

    def get(self):
        return self

    def reset_scope_templates(self):
        """
        Ensure a templates dictionary exists for the current scope in the database
        """
        templates_key = f"verification_templates_{self.scope}"

        self.tools_db.set(templates_key, json.dumps({}))

    def _ensure_scope_templates(self):
        """
        Ensure a templates dictionary exists for the current scope in the database
        """
        templates_key = f"verification_templates_{self.scope}"

        # Check if templates exist for this scope
        templates_exist = self.tools_db.if_exist(templates_key)

        if templates_exist.is_error() and not templates_exist.is_data():
            # Initialize empty templates dictionary if not exists
            self.tools_db.set(templates_key, json.dumps({}))
        else:
            allt = self.get_all_templates()

            for k, v in allt.items():
                if 'name' not in v:
                    continue
                self.tidmp[v['name']] = k

    def add_config_template(self, template: ConfigTemplate) -> str:
        """
        Add a new configuration template to the database

        Args:
            template (ConfigTemplate): The configuration template

        Returns:
            str: Unique identifier of the template
        """
        # Ensure template has the current scope
        template.scope = self.scope

        # Generate a unique template ID
        template_id = secrets.token_urlsafe(8)

        # Get existing templates for this scope
        templates = self.get_all_templates()

        # Add new template
        self.tidmp[template.name] = template_id
        templates[template_id] = asdict(template)

        # Save updated templates back to database
        templates_key = f"verification_templates_{self.scope}"
        save_result = self.tools_db.set(templates_key, json.dumps(templates))

        if save_result.is_error():
            raise ValueError("Could not save template")

        return template_id

    def get_all_templates(self):
        templates_key = f"verification_templates_{self.scope}"
        templates_result = self.tools_db.get(templates_key)

        if not templates_result.is_error() and templates_result.is_data():
            try:
                templates_result.result.data = json.loads(templates_result.get())
            except Exception as e:
                templates_result.print()
                print(f"Errro loding template data curupted : {str(e)}")
                templates_result.result.data = {}
        else:
            templates_result.result.data = {}
        if not isinstance(templates_result, dict):
            templates_result = templates_result.result.data
        return templates_result

    def generate_code(self, template_id: str) -> str:
        """
        Generate a code based on the configuration template

        Args:
            template_id (str): ID of the configuration template

        Returns:
            str: Generated verification code
        """
        # Get templates for this scope
        templates = self.get_all_templates()
        print(templates, self.tidmp, template_id)
        if template_id not in templates:
            template_id = self.tidmp.get(template_id, template_id)
        if template_id not in templates:
            raise ValueError("Invalid configuration template")

        template_dict = templates[template_id]
        ConfigTemplate(**template_dict)

        # Generate a random code with max 16 characters
        code = secrets.token_urlsafe(10)[:16]

        # Prepare code information
        code_info = {
            'template_id': template_id,
            'created_at': time.time(),
            'uses_count': 0,
            'scope': self.scope
        }

        # Store code information in database
        codes_key = f"verification_codes_{self.scope}"
        existing_codes_result = self.tools_db.get(codes_key)

        existing_codes = {}
        if not existing_codes_result.is_error() and existing_codes_result.is_data():
            d = existing_codes_result.get()
            if isinstance(d, list):
                d = d[0]
            existing_codes = json.loads(d)

        existing_codes[code] = code_info

        save_result = self.tools_db.set(codes_key, json.dumps(existing_codes))

        if save_result.is_error():
            raise ValueError("Could not save generated code")

        return code

    def validate_code(self, code: str) -> dict[str, Any] | None:
        """
        Validate a code and return template information

        Args:
            code (str): Code to validate

        Returns:
            Optional[Dict[str, Any]]: Template information for valid code, else None
        """
        # Get codes for this scope
        codes_key = f"verification_codes_{self.scope}"
        codes_result = self.tools_db.get(codes_key)

        if codes_result.is_error() or not codes_result.is_data():
            return None

        d = codes_result.get()
        if isinstance(d, list):
            d = d[0]
        existing_codes = json.loads(d)

        if code not in existing_codes:
            return None

        code_info = existing_codes[code]

        # Check if code is from the same scope
        if code_info.get('scope') != self.scope:
            return None

        # Get templates for this scope
        templates = self.get_all_templates()
        template_id = code_info['template_id']

        if template_id not in templates:
            return templates

        template_dict = templates[template_id]
        template = ConfigTemplate(**template_dict)

        # Check usage count
        if code_info['uses_count'] >= template.max_uses:
            del existing_codes[code]
            self.tools_db.set(codes_key, json.dumps(existing_codes))
            return None

        # Check time validity for timed codes
        if template.usage_type == 'timed':
            current_time = time.time()
            if template.valid_duration and (current_time - code_info['created_at']) > template.valid_duration:
                del existing_codes[code]
                self.tools_db.set(codes_key, json.dumps(existing_codes))
                return None

        # Update uses count
        existing_codes[code]['uses_count'] += 1
        uses_count = existing_codes[code].get('uses_count', 1)
        # Remove code if it's a one-time use
        if template.usage_type == 'one_time':
            del existing_codes[code]

        # Save updated codes
        self.tools_db.set(codes_key, json.dumps(existing_codes))

        return {
            'template_name': template.name,
            'usage_type': template.usage_type,
            'uses_count': uses_count
        }

__init__(tools_db, scope='main')

Initialize VerificationSystem with DB Tools integration

Parameters:

Name	Type	Description	Default
tools_db	Tools	Database tools from toolboxv2.mods.DB	required
scope	str	Scope for templates and codes. Defaults to "main".	'main'

Source code in toolboxv2/mods/CodeVerification.py

def __init__(self, tools_db, scope="main"):
    """
    Initialize VerificationSystem with DB Tools integration

    Args:
        tools_db (Tools): Database tools from toolboxv2.mods.DB
        scope (str, optional): Scope for templates and codes. Defaults to "main".
    """
    self.tools_db = tools_db
    self.scope = scope
    self.tidmp = {}
    self._ensure_scope_templates()

add_config_template(template)

Add a new configuration template to the database

Parameters:

Name	Type	Description	Default
template	ConfigTemplate	The configuration template	required

Returns:

Name	Type	Description
str	str	Unique identifier of the template

Source code in toolboxv2/mods/CodeVerification.py

def add_config_template(self, template: ConfigTemplate) -> str:
    """
    Add a new configuration template to the database

    Args:
        template (ConfigTemplate): The configuration template

    Returns:
        str: Unique identifier of the template
    """
    # Ensure template has the current scope
    template.scope = self.scope

    # Generate a unique template ID
    template_id = secrets.token_urlsafe(8)

    # Get existing templates for this scope
    templates = self.get_all_templates()

    # Add new template
    self.tidmp[template.name] = template_id
    templates[template_id] = asdict(template)

    # Save updated templates back to database
    templates_key = f"verification_templates_{self.scope}"
    save_result = self.tools_db.set(templates_key, json.dumps(templates))

    if save_result.is_error():
        raise ValueError("Could not save template")

    return template_id

generate_code(template_id)

Generate a code based on the configuration template

Parameters:

Name	Type	Description	Default
template_id	str	ID of the configuration template	required

Returns:

Name	Type	Description
str	str	Generated verification code

Source code in toolboxv2/mods/CodeVerification.py

def generate_code(self, template_id: str) -> str:
    """
    Generate a code based on the configuration template

    Args:
        template_id (str): ID of the configuration template

    Returns:
        str: Generated verification code
    """
    # Get templates for this scope
    templates = self.get_all_templates()
    print(templates, self.tidmp, template_id)
    if template_id not in templates:
        template_id = self.tidmp.get(template_id, template_id)
    if template_id not in templates:
        raise ValueError("Invalid configuration template")

    template_dict = templates[template_id]
    ConfigTemplate(**template_dict)

    # Generate a random code with max 16 characters
    code = secrets.token_urlsafe(10)[:16]

    # Prepare code information
    code_info = {
        'template_id': template_id,
        'created_at': time.time(),
        'uses_count': 0,
        'scope': self.scope
    }

    # Store code information in database
    codes_key = f"verification_codes_{self.scope}"
    existing_codes_result = self.tools_db.get(codes_key)

    existing_codes = {}
    if not existing_codes_result.is_error() and existing_codes_result.is_data():
        d = existing_codes_result.get()
        if isinstance(d, list):
            d = d[0]
        existing_codes = json.loads(d)

    existing_codes[code] = code_info

    save_result = self.tools_db.set(codes_key, json.dumps(existing_codes))

    if save_result.is_error():
        raise ValueError("Could not save generated code")

    return code

reset_scope_templates()

Ensure a templates dictionary exists for the current scope in the database

Source code in toolboxv2/mods/CodeVerification.py

def reset_scope_templates(self):
    """
    Ensure a templates dictionary exists for the current scope in the database
    """
    templates_key = f"verification_templates_{self.scope}"

    self.tools_db.set(templates_key, json.dumps({}))

validate_code(code)

Validate a code and return template information

Parameters:

Name	Type	Description	Default
code	str	Code to validate	required

Returns:

Type	Description
dict[str, Any] | None	Optional[Dict[str, Any]]: Template information for valid code, else None

Source code in toolboxv2/mods/CodeVerification.py

def validate_code(self, code: str) -> dict[str, Any] | None:
    """
    Validate a code and return template information

    Args:
        code (str): Code to validate

    Returns:
        Optional[Dict[str, Any]]: Template information for valid code, else None
    """
    # Get codes for this scope
    codes_key = f"verification_codes_{self.scope}"
    codes_result = self.tools_db.get(codes_key)

    if codes_result.is_error() or not codes_result.is_data():
        return None

    d = codes_result.get()
    if isinstance(d, list):
        d = d[0]
    existing_codes = json.loads(d)

    if code not in existing_codes:
        return None

    code_info = existing_codes[code]

    # Check if code is from the same scope
    if code_info.get('scope') != self.scope:
        return None

    # Get templates for this scope
    templates = self.get_all_templates()
    template_id = code_info['template_id']

    if template_id not in templates:
        return templates

    template_dict = templates[template_id]
    template = ConfigTemplate(**template_dict)

    # Check usage count
    if code_info['uses_count'] >= template.max_uses:
        del existing_codes[code]
        self.tools_db.set(codes_key, json.dumps(existing_codes))
        return None

    # Check time validity for timed codes
    if template.usage_type == 'timed':
        current_time = time.time()
        if template.valid_duration and (current_time - code_info['created_at']) > template.valid_duration:
            del existing_codes[code]
            self.tools_db.set(codes_key, json.dumps(existing_codes))
            return None

    # Update uses count
    existing_codes[code]['uses_count'] += 1
    uses_count = existing_codes[code].get('uses_count', 1)
    # Remove code if it's a one-time use
    if template.usage_type == 'one_time':
        del existing_codes[code]

    # Save updated codes
    self.tools_db.set(codes_key, json.dumps(existing_codes))

    return {
        'template_name': template.name,
        'usage_type': template.usage_type,
        'uses_count': uses_count
    }

DB

blob_instance

BlobDB

A persistent, encrypted dictionary-like database that uses the BlobStorage system as its backend, making it networked and fault-tolerant.

This implementation uses multiple BlobFiles instead of a single virtual file. Keys are structured like USER::XYZ::: or MANAGER::SPACE::DATA::XYZ and are converted to file paths by replacing :: with /.

Source code in toolboxv2/mods/DB/blob_instance.py

class BlobDB:
    """
    A persistent, encrypted dictionary-like database that uses the BlobStorage
    system as its backend, making it networked and fault-tolerant.

    This implementation uses multiple BlobFiles instead of a single virtual file.
    Keys are structured like USER::XYZ::: or MANAGER::SPACE::DATA::XYZ and are
    converted to file paths by replacing :: with /.
    """
    auth_type = AuthenticationTypes.location

    def __init__(self):
        self.data: dict = {}  # In-memory cache of all data
        self.key: str | None = None
        self.db_path: str | None = None  # Base path for the database
        self.storage_client: BlobStorage | None = None


    def _key_to_blob_path(self, key: str) -> str:
        """
        Converts a database key to a blob file path.
        Replaces :: with / to create a hierarchical file structure.

        Examples:
            USER::XYZ::: -> db_path/USER/XYZ.json
            MANAGER::SPACE::DATA::XYZ -> db_path/MANAGER/SPACE/DATA/XYZ.json
        """
        # Replace :: with / and remove trailing/leading colons and slashes
        path_parts = key.replace('::', '/').strip('/').strip(':').strip('/')
        # Combine with base db_path
        return f"{self.db_path}/{path_parts}.json"

    def _load_blob_file(self, blob_path: str) -> dict:
        """
        Loads data from a specific blob file.
        Returns empty dict if file doesn't exist.
        """
        try:
            db_file = BlobFile(blob_path, mode='r', storage=self.storage_client, key=self.key)
            if not db_file.exists():
                return {}
            with db_file as f:
                data = f.read_json()
                return data if data else {}
        except Exception as e:
            print(f"Warning: Could not load blob file '{blob_path}'. Error: {e}")
            return {}

    def _save_blob_file(self, blob_path: str, data: dict) -> bool:
        """
        Saves data to a specific blob file.
        Returns True on success, False on failure.
        """
        try:
            # Ensure the blob exists first
            db_file = BlobFile(blob_path, mode='r', storage=self.storage_client, key=self.key)
            if not db_file.exists():
                db_file.create()

            with BlobFile(blob_path, mode='w', storage=self.storage_client, key=self.key) as f:
                f.write_json(data)
            return True
        except Exception as e:
            print(f"Error: Could not save blob file '{blob_path}'. Error: {e}")
            return False

    def initialize(self, db_path: str, key: str, storage_client: BlobStorage) -> Result:
        """
        Initializes the database from a location within the blob storage.

        Args:
            db_path (str): The base path within the blob storage,
                           e.g., "my_database_blob".
            key (str): The encryption key for the database content.
            storage_client (BlobStorage): An initialized BlobStorage client instance.

        Returns:
            Result: An OK result if successful.
        """
        self.db_path = db_path.rstrip('/')
        self.key = key
        self.storage_client = storage_client

        print(f"Initializing BlobDB with base path: '{self.db_path}'...")

        # Initialize with empty data - data will be loaded on-demand
        self.data = {}

        print("Successfully initialized database with multi-file storage.")
        return Result.ok().set_origin("Blob Dict DB")

    def exit(self) -> Result:
        """
        Saves the current state of the database back to the blob storage.
        Each key is saved to its own blob file based on the key structure.
        """
        print("BLOB DB on exit ", not all([self.key, self.db_path, self.storage_client]))
        if not all([self.key, self.db_path, self.storage_client]):
            return Result.default_internal_error(
                info="Database not initialized. Cannot exit."
            ).set_origin("Blob Dict DB")

        print(f"Saving database to blob storage at base path: '{self.db_path}'...")

        errors = []
        saved_count = 0

        try:
            # Group keys by their blob file path
            blob_files_data = {}
            for key, value in self.data.items():
                blob_path = self._key_to_blob_path(key)
                if blob_path not in blob_files_data:
                    blob_files_data[blob_path] = {}
                blob_files_data[blob_path][key] = value

            # Save each blob file
            for blob_path, data in blob_files_data.items():
                if self._save_blob_file(blob_path, data):
                    saved_count += 1
                else:
                    errors.append(f"Failed to save {blob_path}")

            if errors:
                return Result.custom_error(
                    data=errors,
                    info=f"Saved {saved_count} blob files, but {len(errors)} failed: {errors}"
                ).set_origin("Blob Dict DB")

            print(f"Success: Database saved to {saved_count} blob file(s).")
            return Result.ok().set_origin("Blob Dict DB")

        except Exception as e:
            return Result.custom_error(
                data=e,
                info=f"Error saving database to blob storage: {e}"
            ).set_origin("Blob Dict DB")

    # --- Data Manipulation Methods ---
    # These methods now handle loading data on-demand from multiple blob files

    def _ensure_key_loaded(self, key: str):
        """
        Ensures that data for a specific key is loaded into memory.
        If the key contains wildcards, loads all matching blob files.
        """
        if key.endswith('*'):
            # For wildcard searches, we need to scan and load relevant files
            # This is a simplified approach - in production you might want to maintain an index
            prefix = key.replace('*', '')
            # For now, we'll rely on the in-memory cache
            # A more sophisticated approach would scan blob storage
            return

        # Check if key is already in memory
        if key in self.data:
            return

        # Load the blob file for this key
        blob_path = self._key_to_blob_path(key)
        blob_data = self._load_blob_file(blob_path)

        # Merge loaded data into memory
        self.data.update(blob_data)

    def get(self, key: str) -> Result:
        data = []

        if key == 'all':
            # Load all data - this is expensive, consider pagination in production
            data_info = "Returning all data available"
            data = list(self.data.items())
        elif key == "all-k":
            data_info = "Returning all keys"
            data = list(self.data.keys())
        else:
            # Ensure the key or matching keys are loaded
            self._ensure_key_loaded(key)
            data_info = f"Returning values for keys starting with '{key.replace('*', '')}'"
            data = [self.data[k] for k in self.scan_iter(key)]

        if not data:
            return Result.default_internal_error(info=f"No data found for key '{key}'").set_origin("Blob Dict DB")

        return Result.ok(data=data, data_info=data_info).set_origin("Blob Dict DB")

    def set(self, key: str, value) -> Result:
        if not isinstance(key, str) or not key:
            return Result.default_user_error(info="Key must be a non-empty string.").set_origin("Blob Dict DB")

        # Ensure existing data for this key is loaded first
        self._ensure_key_loaded(key)

        self.data[key] = value
        return Result.ok().set_origin("Blob Dict DB")

    def scan_iter(self, search: str = ''):
        if not self.data:
            return []
        prefix = search.replace('*', '')
        return [key for key in self.data if key.startswith(prefix)]

    def append_on_set(self, key: str, value: list) -> Result:
        # Ensure existing data for this key is loaded first
        self._ensure_key_loaded(key)

        if key not in self.data:
            self.data[key] = []

        if not isinstance(self.data[key], list):
            return Result.default_user_error(info=f"Existing value for key '{key}' is not a list.").set_origin(
                "Blob Dict DB")

        # Use a set for efficient checking to avoid duplicates
        existing_set = set(self.data[key])
        new_items = [item for item in value if item not in existing_set]
        self.data[key].extend(new_items)
        return Result.ok().set_origin("Blob Dict DB")

    def if_exist(self, key: str) -> int:
        # Ensure data is loaded for existence check
        self._ensure_key_loaded(key)

        if key.endswith('*'):
            return len(self.scan_iter(key))
        return 1 if key in self.data else 0

    def delete(self, key: str, matching: bool = False) -> Result:
        # Ensure data is loaded before deletion
        self._ensure_key_loaded(key)

        keys_to_delete = []
        if matching:
            keys_to_delete = self.scan_iter(key)
        elif key in self.data:
            keys_to_delete.append(key)

        if not keys_to_delete:
            return Result.default_internal_error(info=f"No keys found to delete for pattern '{key}'").set_origin(
                "Blob Dict DB")

        deleted_items = {k: self.data.pop(k) for k in keys_to_delete}
        return Result.ok(
            data=list(deleted_items.items()),
            data_info=f"Successfully removed {len(deleted_items)} item(s)."
        ).set_origin("Blob Dict DB")

exit()

Saves the current state of the database back to the blob storage. Each key is saved to its own blob file based on the key structure.

Source code in toolboxv2/mods/DB/blob_instance.py

def exit(self) -> Result:
    """
    Saves the current state of the database back to the blob storage.
    Each key is saved to its own blob file based on the key structure.
    """
    print("BLOB DB on exit ", not all([self.key, self.db_path, self.storage_client]))
    if not all([self.key, self.db_path, self.storage_client]):
        return Result.default_internal_error(
            info="Database not initialized. Cannot exit."
        ).set_origin("Blob Dict DB")

    print(f"Saving database to blob storage at base path: '{self.db_path}'...")

    errors = []
    saved_count = 0

    try:
        # Group keys by their blob file path
        blob_files_data = {}
        for key, value in self.data.items():
            blob_path = self._key_to_blob_path(key)
            if blob_path not in blob_files_data:
                blob_files_data[blob_path] = {}
            blob_files_data[blob_path][key] = value

        # Save each blob file
        for blob_path, data in blob_files_data.items():
            if self._save_blob_file(blob_path, data):
                saved_count += 1
            else:
                errors.append(f"Failed to save {blob_path}")

        if errors:
            return Result.custom_error(
                data=errors,
                info=f"Saved {saved_count} blob files, but {len(errors)} failed: {errors}"
            ).set_origin("Blob Dict DB")

        print(f"Success: Database saved to {saved_count} blob file(s).")
        return Result.ok().set_origin("Blob Dict DB")

    except Exception as e:
        return Result.custom_error(
            data=e,
            info=f"Error saving database to blob storage: {e}"
        ).set_origin("Blob Dict DB")

initialize(db_path, key, storage_client)

Initializes the database from a location within the blob storage.

Parameters:

Name	Type	Description	Default
db_path	str	The base path within the blob storage, e.g., "my_database_blob".	required
key	str	The encryption key for the database content.	required
storage_client	BlobStorage	An initialized BlobStorage client instance.	required

Returns:

Name	Type	Description
Result	Result	An OK result if successful.

Source code in toolboxv2/mods/DB/blob_instance.py

def initialize(self, db_path: str, key: str, storage_client: BlobStorage) -> Result:
    """
    Initializes the database from a location within the blob storage.

    Args:
        db_path (str): The base path within the blob storage,
                       e.g., "my_database_blob".
        key (str): The encryption key for the database content.
        storage_client (BlobStorage): An initialized BlobStorage client instance.

    Returns:
        Result: An OK result if successful.
    """
    self.db_path = db_path.rstrip('/')
    self.key = key
    self.storage_client = storage_client

    print(f"Initializing BlobDB with base path: '{self.db_path}'...")

    # Initialize with empty data - data will be loaded on-demand
    self.data = {}

    print("Successfully initialized database with multi-file storage.")
    return Result.ok().set_origin("Blob Dict DB")

local_instance

load_from_json(filename)

Lädt Daten aus einer JSON-Datei.

:param filename: Der Dateiname oder Pfad der zu ladenden Datei. :return: Die geladenen Daten.

Source code in toolboxv2/mods/DB/local_instance.py

def load_from_json(filename):
    """
    Lädt Daten aus einer JSON-Datei.

    :param filename: Der Dateiname oder Pfad der zu ladenden Datei.
    :return: Die geladenen Daten.
    """
    if not os.path.exists(filename):
        return {'data': ''}

    with open(filename) as file:
        return json.load(file)

save_to_json(data, filename)

Speichert die übergebenen Daten in einer JSON-Datei.

:param data: Die zu speichernden Daten. :param filename: Der Dateiname oder Pfad, in dem die Daten gespeichert werden sollen.

Source code in toolboxv2/mods/DB/local_instance.py

def save_to_json(data, filename):
    """
    Speichert die übergebenen Daten in einer JSON-Datei.

    :param data: Die zu speichernden Daten.
    :param filename: Der Dateiname oder Pfad, in dem die Daten gespeichert werden sollen.
    """
    if not os.path.exists(filename):
        open(filename, 'a').close()

    with open(filename, 'w+') as file:
        json.dump(data, file, indent=4)

reddis_instance

sync_redis_databases(source_url, target_url)

Synchronize keys from the source Redis database to the target Redis database. This function scans all keys in the source DB and uses DUMP/RESTORE to replicate data to the target.

Parameters:

Name	Type	Description	Default
source_url	str	The Redis URL of the source database.	required
target_url	str	The Redis URL of the target database.	required

Returns:

Name	Type	Description
int		The number of keys successfully synchronized.

Source code in toolboxv2/mods/DB/reddis_instance.py

def sync_redis_databases(source_url, target_url):
    """Synchronize keys from the source Redis database to the target Redis database.
    This function scans all keys in the source DB and uses DUMP/RESTORE to replicate data to the target.

    Args:
        source_url (str): The Redis URL of the source database.
        target_url (str): The Redis URL of the target database.

    Returns:
        int: The number of keys successfully synchronized.
    """
    try:
        src_client = redis.from_url(source_url)
        tgt_client = redis.from_url(target_url)
    except Exception as e:
        print(f"Error connecting to one of the Redis instances: {e}")
        return 0

    total_synced = 0
    cursor = 0
    try:
        while True:
            cursor, keys = src_client.scan(cursor=cursor, count=100)
            for key in keys:
                try:
                    serialized_value = src_client.dump(key)
                    if serialized_value is None:
                        continue
                    # Restore key with TTL=0 and replace existing key
                    tgt_client.restore(key, 0, serialized_value, replace=True)
                    total_synced += 1
                except Exception as e:
                    print(f"Error syncing key {key}: {e}")
            if cursor == 0:
                break
    except Exception as scan_error:
        print(f"Error during scanning keys: {scan_error}")

    print(f"Synced {total_synced} keys from {source_url} to {target_url}")
    return total_synced

tb_adapter

DB

Bases: ABC

Source code in toolboxv2/mods/DB/tb_adapter.py

class DB(ABC):
    @abc.abstractmethod
    def get(self, query: str) -> Result:
        """get data"""

    @abc.abstractmethod
    def set(self, query: str, value) -> Result:
        """set data"""

    @abc.abstractmethod
    def append_on_set(self, query: str, value) -> Result:
        """append set data"""

    @abc.abstractmethod
    def delete(self, query: str, matching=False) -> Result:
        """delete data"""

    @abc.abstractmethod
    def if_exist(self, query: str) -> bool:
        """return True if query exists"""

    @abc.abstractmethod
    def exit(self) -> Result:
        """Close DB connection and optional save data"""

append_on_set(query, value) abstractmethod

append set data

Source code in toolboxv2/mods/DB/tb_adapter.py

@abc.abstractmethod
def append_on_set(self, query: str, value) -> Result:
    """append set data"""

delete(query, matching=False) abstractmethod

delete data

Source code in toolboxv2/mods/DB/tb_adapter.py

@abc.abstractmethod
def delete(self, query: str, matching=False) -> Result:
    """delete data"""

exit() abstractmethod

Close DB connection and optional save data

Source code in toolboxv2/mods/DB/tb_adapter.py

@abc.abstractmethod
def exit(self) -> Result:
    """Close DB connection and optional save data"""

get(query) abstractmethod

get data

Source code in toolboxv2/mods/DB/tb_adapter.py

@abc.abstractmethod
def get(self, query: str) -> Result:
    """get data"""

if_exist(query) abstractmethod

return True if query exists

Source code in toolboxv2/mods/DB/tb_adapter.py

@abc.abstractmethod
def if_exist(self, query: str) -> bool:
    """return True if query exists"""

set(query, value) abstractmethod

set data

Source code in toolboxv2/mods/DB/tb_adapter.py

@abc.abstractmethod
def set(self, query: str, value) -> Result:
    """set data"""

ui

api_change_mode(self, request) async

Changes the database mode from a JSON POST body.

Source code in toolboxv2/mods/DB/ui.py

@export(mod_name=Name, name="api_change_mode", api=True, api_methods=['POST'], request_as_kwarg=True)
async def api_change_mode(self, request: RequestData):
    """Changes the database mode from a JSON POST body."""
    data = request.body
    if not data or "mode" not in data:
        return Result.default_user_error("Request body must contain 'mode'.")
    new_mode = data.get("mode", "LC")
    return self.edit_programmable(DatabaseModes.crate(new_mode))

api_delete_key(self, request) async

Deletes a key from a JSON POST body.

Source code in toolboxv2/mods/DB/ui.py

@export(mod_name=Name, name="api_delete_key", api=True, api_methods=['POST'], request_as_kwarg=True)
async def api_delete_key(self, request: RequestData):
    """Deletes a key from a JSON POST body."""
    data = request.body
    if not data or 'key' not in data:
        return Result.default_user_error("Request body must contain 'key'.")
    key = data['key']
    if not key:
        return Result.default_user_error("Key parameter is required.")
    return self.delete(key)

api_get_all_keys(self, request) async

Returns a list of all keys in the database.

Source code in toolboxv2/mods/DB/ui.py

@export(mod_name=Name, name="api_get_all_keys", api=True, request_as_kwarg=True)
async def api_get_all_keys(self, request: RequestData):
    """Returns a list of all keys in the database."""
    if self.data_base:
        keys_result = self.data_base.get('all-k')
        if keys_result.is_error():
            return keys_result

        unwrapped_keys = _unwrap_data(keys_result.get())
        if not isinstance(unwrapped_keys, list):
            self.app.logger.warning(f"get_all_keys did not return a list. Got: {type(unwrapped_keys)}")
            return Result.json(data=[])

        return Result.json(data=sorted(unwrapped_keys))
    return Result.default_internal_error("DB not initialized")

api_get_blob_status(self, request) async

Returns blob storage status - admin/trusted only.

Source code in toolboxv2/mods/DB/ui.py

@export(mod_name=Name, name="api_get_blob_status", api=True, request_as_kwarg=True)
async def api_get_blob_status(self, request: RequestData):
    """Returns blob storage status - admin/trusted only."""
    if not await _is_admin_or_trusted(self.app, request):
        return Result.default_user_error("Access denied")

    try:
        blob_storage = self.app.root_blob_storage
        if not blob_storage:
            return Result.json(data={"status": "unavailable", "servers": []})

        # Get server status
        servers_status = []
        for server in blob_storage.servers:
            try:
                # Basic health check
                status = "online" if server else "offline"
                servers_status.append({
                    "address": str(server),
                    "status": status
                })
            except Exception:
                servers_status.append({
                    "address": str(server),
                    "status": "error"
                })

        return Result.json(data={
            "status": "available",
            "servers": servers_status,
            "storage_dir": getattr(blob_storage, 'storage_directory', 'unknown')
        })
    except Exception as e:
        return Result.default_internal_error(f"Blob status error: {str(e)}")

api_get_cluster_status(self, request) async

Get DB cluster status - admin/trusted only.

Source code in toolboxv2/mods/DB/ui.py

@export(mod_name=Name, name="api_get_cluster_status", api=True, request_as_kwarg=True)
async def api_get_cluster_status(self, request: RequestData):
    """Get DB cluster status - admin/trusted only."""
    if not await _is_admin_or_trusted(self.app, request):
        return Result.default_user_error("Access denied")

    try:
        from toolboxv2.utils.clis.db_cli_manager import ClusterManager
        manager = ClusterManager()
        online_list, server_list = manager.status_all(silent=True)

        instances = []
        for instance_id, instance in manager.instances.items():
            pid, version = instance.read_state()
            instances.append({
                "id": instance_id,
                "port": instance.port,
                "host": instance.host,
                "status": "online" if pid else "offline",
                "pid": pid,
                "version": version
            })

        return Result.json(data={
            "instances": instances,
            "online_count": len(online_list),
            "total_count": len(server_list)
        })
    except Exception as e:
        return Result.default_internal_error(f"Cluster status error: {str(e)}")

api_get_status(self, request) async

Returns the current status of the DB manager.

Source code in toolboxv2/mods/DB/ui.py

@export(mod_name=Name, name="api_get_status", api=True, request_as_kwarg=True)
async def api_get_status(self, request: RequestData):
    """Returns the current status of the DB manager."""
    return Result.json(data={"mode": self.mode})

api_get_value(self, request, key) async

Gets a value for a key and returns it as JSON-friendly text.

Source code in toolboxv2/mods/DB/ui.py

@export(mod_name=Name, name="api_get_value", api=True, request_as_kwarg=True)
async def api_get_value(self, request: RequestData, key: str):
    """Gets a value for a key and returns it as JSON-friendly text."""
    if not key:
        return Result.default_user_error("Key parameter is required.")
    value_res = self.get(key)
    if value_res.is_error():
        return value_res

    value_unwrapped = _unwrap_data(value_res.get())

    if isinstance(value_unwrapped, bytes):
        try:
            value_str = value_unwrapped.decode('utf-8')
        except UnicodeDecodeError:
            value_str = str(value_unwrapped)
    else:
        value_str = str(value_unwrapped)

    # Simplified for a JSON-focused UI. The client will handle formatting.
    return Result.json(data={"key": key, "value": value_str})

api_list_blob_files(self, request) async

List blob files - admin/trusted only.

Source code in toolboxv2/mods/DB/ui.py

@export(mod_name=Name, name="api_list_blob_files", api=True, request_as_kwarg=True)
async def api_list_blob_files(self, request: RequestData):
    """List blob files - admin/trusted only."""
    if not await _is_admin_or_trusted(self.app, request):
        return Result.default_user_error("Access denied")

    try:
        blob_storage = self.app.root_blob_storage
        if not blob_storage:
            return Result.json(data=[])

        # Get blob IDs
        blob_ids = blob_storage.list_blobs()
        blob_files = []

        for blob_id in blob_ids[:100]:  # Limit to first 100
            try:
                info = blob_storage.get_blob_info(blob_id)
                blob_files.append({
                    "id": blob_id,
                    "size": info.get("size", 0),
                    "created": info.get("created", "unknown"),
                    "encrypted": info.get("encrypted", False)
                })
            except Exception:
                blob_files.append({
                    "id": blob_id,
                    "size": 0,
                    "created": "unknown",
                    "encrypted": False
                })

        return Result.json(data=blob_files)
    except Exception as e:
        return Result.default_internal_error(f"Blob listing error: {str(e)}")

api_manage_cluster(self, request) async

Manage cluster instances - admin/trusted only.

Source code in toolboxv2/mods/DB/ui.py

@export(mod_name=Name, name="api_manage_cluster", api=True, api_methods=['POST'], request_as_kwarg=True)
async def api_manage_cluster(self, request: RequestData):
    """Manage cluster instances - admin/trusted only."""
    if not await _is_admin_or_trusted(self.app, request):
        return Result.default_user_error("Access denied")

    data = request.body
    if not data or 'action' not in data:
        return Result.default_user_error("Request body must contain 'action'.")

    action = data['action']
    instance_id = data.get('instance_id')

    try:
        from toolboxv2.utils.clis.db_cli_manager import ClusterManager, get_executable_path
        manager = ClusterManager()

        if action == 'start':
            executable_path = get_executable_path()
            if instance_id:
                result = manager.start(executable_path, "current", instance_id)
            else:
                result = manager.start_all(executable_path, "current")
            return Result.ok(data=f"Start command executed: {result}")

        elif action == 'stop':
            if instance_id:
                result = manager.stop(instance_id)
            else:
                result = manager.stop_all()
            return Result.ok(data=f"Stop command executed: {result}")

        elif action == 'restart':
            if instance_id:
                manager.stop(instance_id)
                executable_path = get_executable_path()
                result = manager.start(executable_path, "current", instance_id)
            else:
                manager.stop_all()
                executable_path = get_executable_path()
                result = manager.start_all(executable_path, "current")
            return Result.ok(data=f"Restart command executed: {result}")

        else:
            return Result.default_user_error("Invalid action")

    except Exception as e:
        return Result.default_internal_error(f"Cluster management error: {str(e)}")

api_set_value(self, request) async

Sets a key-value pair from a JSON POST body.

Source code in toolboxv2/mods/DB/ui.py

@export(mod_name=Name, name="api_set_value", api=True, api_methods=['POST'], request_as_kwarg=True)
async def api_set_value(self, request: RequestData):
    """Sets a key-value pair from a JSON POST body."""
    data = request.body
    if not data or 'key' not in data or 'value' not in data:
        return Result.default_user_error("Request body must contain 'key' and 'value'.")
    key = data['key']
    value = data['value']
    if not key:
        return Result.default_user_error("Key cannot be empty.")
    return self.set(key, value)

db_manager_ui(**kwargs)

Serves the refactored, JSON-focused UI for the DB Manager.

Source code in toolboxv2/mods/DB/ui.py

@export(mod_name=Name, name="ui", api=True, state=False)
def db_manager_ui(**kwargs):
    """Serves the refactored, JSON-focused UI for the DB Manager."""
    html_content = """
    <!DOCTYPE html>
    <html lang="en">
    <head>
        <meta charset="UTF-8">
        <meta name="viewport" content="width=device-width, initial-scale=1.0">
        <title>DB Manager</title>
        <style>
            :root {
                --font-family-sans: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif;
                --font-family-mono: "SF Mono", "Menlo", "Monaco", "Courier New", Courier, monospace;
                --color-bg: #f8f9fa;
                --color-panel-bg: #ffffff;
                --color-border: #dee2e6;
                --color-text: #212529;
                --color-text-muted: #6c757d;
                --color-primary: #0d6efd;
                --color-primary-hover: #0b5ed7;
                --color-danger: #dc3545;
                --color-danger-hover: #bb2d3b;
                --color-key-folder-icon: #f7b731;
                --color-key-file-icon: #adb5bd;
                --color-key-hover-bg: #e9ecef;
                --color-key-selected-bg: #0d6efd;
                --color-key-selected-text: #ffffff;
                --shadow-sm: 0 1px 2px 0 rgba(0, 0, 0, 0.05);
                --radius: 0.375rem;
            }

            /* Basic styles */
            * { box-sizing: border-box; }
            html { font-size: 16px; }

            body {
                font-family: var(--font-family-sans);
                background-color: var(--color-bg);
                color: var(--color-text);
                margin: 0;
                padding: 1rem;
                display: flex;
                flex-direction: column;
                height: 100vh;
            }

            /* Main layout */
            .db-manager-container { display: flex; flex-direction: column; height: 100%; gap: 1rem; }
            .db-header { display: flex; justify-content: space-between; align-items: center; padding-bottom: 1rem; border-bottom: 1px solid var(--color-border); flex-shrink: 0; }
            .db-main-content { display: flex; gap: 1rem; flex: 1; min-height: 0; }

            /* Panels */
            .db-panel { background-color: var(--color-panel-bg); border: 1px solid var(--color-border); border-radius: var(--radius); box-shadow: var(--shadow-sm); display: flex; flex-direction: column; min-height: 0; }
            .key-panel { width: 350px; min-width: 250px; max-width: 450px; }
            .editor-panel, .placeholder-panel { flex-grow: 1; }
            .panel-header { display: flex; justify-content: space-between; align-items: center; padding: 0.75rem 1rem; border-bottom: 1px solid var(--color-border); flex-shrink: 0; }
            .panel-header h2 { font-size: 1.1rem; margin: 0; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; }

            /* Controls */
            select, input[type="text"], textarea, button { font-size: 1rem; }
            select, input[type="text"] { background-color: var(--color-bg); color: var(--color-text); border: 1px solid var(--color-border); border-radius: var(--radius); padding: 0.5rem 0.75rem; }
            select:focus, input[type="text"]:focus, textarea:focus { outline: 2px solid var(--color-primary); outline-offset: -1px; }
            button { border: none; border-radius: var(--radius); padding: 0.5rem 1rem; font-weight: 500; cursor: pointer; transition: background-color 0.2s; }
            button.primary { background-color: var(--color-primary); color: white; }
            button.primary:hover { background-color: var(--color-primary-hover); }
            button.danger { background-color: var(--color-danger); color: white; }
            button.danger:hover { background-color: var(--color-danger-hover); }
            .header-actions { display: flex; gap: 0.5rem; }

            /* Key Tree View */
            #keySearchInput { width: calc(100% - 2rem); margin: 1rem; flex-shrink: 0; }
            .key-tree-container { font-family: var(--font-family-mono); font-size: 0.9rem; padding: 0 0.5rem 1rem; overflow-y: auto; flex: 1; min-height: 0; }
            .key-tree-container ul { list-style: none; padding-left: 0; margin: 0; }
            .key-tree-container li { padding-left: 20px; position: relative; }
            .node-label { display: flex; align-items: center; padding: 4px 8px; cursor: pointer; border-radius: 4px; word-break: break-all; user-select: none; }
            .node-label:hover { background-color: var(--color-key-hover-bg); }
            .node-label.selected { background-color: var(--color-key-selected-bg); color: var(--color-key-selected-text); }
            .node-label.selected .node-icon { color: var(--color-key-selected-text) !important; }
            .node-icon { width: 20px; text-align: center; margin-right: 5px; flex-shrink: 0; }
            .tree-folder > .node-label .node-icon { color: var(--color-key-folder-icon); font-style: normal; }
            .tree-folder > .node-label .node-icon::before { content: '▸'; display: inline-block; transition: transform 0.15s ease-in-out; }
            .tree-folder.open > .node-label .node-icon::before { transform: rotate(90deg); }
            .tree-leaf > .node-label .node-icon { color: var(--color-key-file-icon); }
            .tree-leaf > .node-label .node-icon::before { content: '•'; }
            .tree-children { display: none; }
            .tree-folder.open > .tree-children { display: block; }

            /* Editor Panel */
            .editor-toolbar { display: flex; gap: 1rem; align-items: center; padding: 0.75rem 1rem; border-bottom: 1px solid var(--color-border); flex-shrink: 0; }
            #valueEditor { flex: 1; width: 100%; min-height: 0; border: none; resize: none; font-family: var(--font-family-mono); font-size: 0.95rem; line-height: 1.5; padding: 1rem; background: transparent; color: var(--color-text); }
            #valueEditor:focus { outline: none; }

            /* Placeholder and Utility */
            .placeholder-panel { display: flex; flex-direction: column; align-items: center; justify-content: center; color: var(--color-text-muted); text-align: center; }
            .hidden { display: none !important; }
            .key-tree-container p.status-message { padding: 1rem; margin: 0; color: var(--color-text-muted); text-align: center; }

            /* Custom Scrollbars */
            .key-tree-container::-webkit-scrollbar, #valueEditor::-webkit-scrollbar { width: 8px; height: 8px; }
            .key-tree-container::-webkit-scrollbar-track, #valueEditor::-webkit-scrollbar-track { background: transparent; }
            .key-tree-container::-webkit-scrollbar-thumb, #valueEditor::-webkit-scrollbar-thumb { background-color: var(--color-border); border-radius: 4px; }
            .key-tree-container::-webkit-scrollbar-thumb:hover, #valueEditor::-webkit-scrollbar-thumb:hover { background-color: var(--color-text-muted); }
            #valueEditor::-webkit-scrollbar-corner { background: transparent; }

            /* Responsive */
            @media (max-width: 768px) {
                body { padding: 0.5rem; }
                .db-main-content { flex-direction: column; }
                .key-panel { width: 100%; max-height: 40vh; }
            }

            /* New styles for enhanced features */
            .tab-container {
                display: flex;
                border-bottom: 1px solid var(--color-border);
                margin-bottom: 1rem;
            }

            .tab-button {
                padding: 0.75rem 1.5rem;
                border: none;
                background: none;
                cursor: pointer;
                border-bottom: 2px solid transparent;
                font-weight: 500;
            }

            .tab-button.active {
                border-bottom-color: var(--color-primary);
                color: var(--color-primary);
            }

            .tab-content {
                display: none;
            }

            .tab-content.active {
                display: block;
            }

            .status-grid {
                display: grid;
                grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
                gap: 1rem;
                margin-bottom: 1rem;
            }

            .status-card {
                background: var(--color-panel-bg);
                border: 1px solid var(--color-border);
                border-radius: var(--radius);
                padding: 1rem;
            }

            .status-indicator {
                display: inline-block;
                width: 8px;
                height: 8px;
                border-radius: 50%;
                margin-right: 0.5rem;
            }

            .status-online { background-color: #28a745; }
            .status-offline { background-color: #dc3545; }
            .status-error { background-color: #ffc107; }

            .blob-file-list {
                max-height: 400px;
                overflow-y: auto;
                border: 1px solid var(--color-border);
                border-radius: var(--radius);
            }

            .blob-file-item {
                display: flex;
                justify-content: space-between;
                align-items: center;
                padding: 0.75rem;
                border-bottom: 1px solid var(--color-border);
            }

            .blob-file-item:last-child {
                border-bottom: none;
            }

            .cluster-controls {
                display: flex;
                gap: 0.5rem;
                margin-bottom: 1rem;
            }

            .instance-list {
                display: grid;
                gap: 0.5rem;
            }

            .instance-item {
                display: flex;
                justify-content: space-between;
                align-items: center;
                padding: 0.75rem;
                background: var(--color-panel-bg);
                border: 1px solid var(--color-border);
                border-radius: var(--radius);
            }
        </style>
    </head>
    <body>
        <div id="dbManagerContainer" class="db-manager-container">
            <header class="db-header">
                <h1>DB Manager</h1>
                <div class="db-mode-selector">
                    <label for="modeSelect">Mode:</label>
                    <select id="modeSelect">
                        <option value="LC">Local Dict</option>
                        <option value="CB">Cloud Blob</option>
                        <option value="LR">Local Redis</option>
                        <option value="RR">Remote Redis</option>
                    </select>
                </div>
            </header>

            <div class="tab-container">
                <button class="tab-button active" data-tab="database">Database</button>
                <button class="tab-button" data-tab="blob-storage" id="blobTab" style="display:none;">Blob Storage</button>
                <button class="tab-button" data-tab="cluster" id="clusterTab" style="display:none;">Cluster</button>
            </div>

            <main class="db-main-content">
                <!-- Database Tab -->
                <div id="database-tab" class="tab-content active">
                    <aside id="keyPanel" class="db-panel key-panel">
                        <div class="panel-header">
                            <h2>Keys</h2>
                            <div class="header-actions">
                                <button id="addKeyBtn" title="Add New Key">+</button>
                                <button id="refreshKeysBtn" title="Refresh Keys">🔄</button>
                            </div>
                        </div>
                        <input type="text" id="keySearchInput" placeholder="Search keys...">
                        <div id="keyTreeContainer" class="key-tree-container"></div>
                    </aside>
                    <section id="editorPanel" class="db-panel editor-panel hidden">
                        <div class="panel-header">
                            <h2 id="selectedKey"></h2>
                            <div class="header-actions">
                                <button id="saveBtn" class="primary">Save</button>
                                <button id="deleteBtn" class="danger">Delete</button>
                            </div>
                        </div>
                        <div class="editor-toolbar">
                            <button id="formatBtn">Format JSON</button>
                        </div>
                        <textarea id="valueEditor" placeholder="Select a key to view its value..."></textarea>
                    </section>
                    <section id="placeholderPanel" class="db-panel editor-panel placeholder-panel">
                        <h3>Select a key to get started</h3>
                        <p>Or click the '+' button to add a new one.</p>
                    </section>
                </div>

                <!-- Blob Storage Tab -->
                <div id="blob-storage-tab" class="tab-content">
                    <div class="status-grid">
                        <div class="status-card">
                            <h3>Blob Storage Status</h3>
                            <div id="blobStorageStatus">Loading...</div>
                        </div>
                        <div class="status-card">
                            <h3>Server Health</h3>
                            <div id="serverHealth">Loading...</div>
                        </div>
                    </div>
                    <div class="db-panel">
                        <div class="panel-header">
                            <h2>Blob Files</h2>
                            <div class="header-actions">
                                <button id="refreshBlobsBtn">🔄 Refresh</button>
                            </div>
                        </div>
                        <div id="blobFileList" class="blob-file-list">Loading...</div>
                    </div>
                </div>

                <!-- Cluster Tab -->
                <div id="cluster-tab" class="tab-content">
                    <div class="cluster-controls">
                        <button id="startAllBtn" class="primary">Start All</button>
                        <button id="stopAllBtn" class="danger">Stop All</button>
                        <button id="restartAllBtn">Restart All</button>
                        <button id="refreshClusterBtn">🔄 Refresh</button>
                    </div>
                    <div class="db-panel">
                        <div class="panel-header">
                            <h2>Cluster Instances</h2>
                        </div>
                        <div id="instanceList" class="instance-list">Loading...</div>
                    </div>
                </div>
            </main>
        </div>
        <script>
        (() => {
            "use strict";
            const API_NAME = "DB";
            let isAdminUser = false;

            class DBManager {
                constructor() {
                    this.cache = {
                        keys: [],
                        selectedKey: null,
                        blobFiles: [],
                        clusterStatus: null
                    };
                    this.dom = {
                        modeSelect: document.getElementById('modeSelect'),
                        keySearchInput: document.getElementById('keySearchInput'),
                        keyTreeContainer: document.getElementById('keyTreeContainer'),
                        editorPanel: document.getElementById('editorPanel'),
                        placeholderPanel: document.getElementById('placeholderPanel'),
                        selectedKey: document.getElementById('selectedKey'),
                        valueEditor: document.getElementById('valueEditor'),
                        addKeyBtn: document.getElementById('addKeyBtn'),
                        refreshKeysBtn: document.getElementById('refreshKeysBtn'),
                        saveBtn: document.getElementById('saveBtn'),
                        deleteBtn: document.getElementById('deleteBtn'),
                        formatBtn: document.getElementById('formatBtn'),
                        tabButtons: document.querySelectorAll('.tab-button'),
                        tabContents: document.querySelectorAll('.tab-content'),
                        blobTab: document.getElementById('blobTab'),
                        clusterTab: document.getElementById('clusterTab'),
                        refreshBlobsBtn: document.getElementById('refreshBlobsBtn'),
                        blobFileList: document.getElementById('blobFileList'),
                        blobStorageStatus: document.getElementById('blobStorageStatus'),
                        serverHealth: document.getElementById('serverHealth'),
                        instanceList: document.getElementById('instanceList'),
                        startAllBtn: document.getElementById('startAllBtn'),
                        stopAllBtn: document.getElementById('stopAllBtn'),
                        restartAllBtn: document.getElementById('restartAllBtn'),
                        refreshClusterBtn: document.getElementById('refreshClusterBtn')
                    };
                    this.init();
                }

                async init() {
                    this.addEventListeners();
                    await this.checkAdminAccess();
                    await this.loadInitialStatus();
                    await this.loadKeys();
                }

                async checkAdminAccess() {
                    try {
                        const res = await this.apiRequest('api_get_blob_status', null, 'GET');
                        if (!res.error) {
                            isAdminUser = true;
                            this.dom.blobTab.style.display = 'block';
                            this.dom.clusterTab.style.display = 'block';
                        }
                    } catch (e) {
                        // User doesn't have admin access
                        isAdminUser = false;
                    }
                }

                addEventListeners() {
                    // Tab switching
                    this.dom.tabButtons.forEach(button => {
                        button.addEventListener('click', (e) => {
                            const tabName = e.target.dataset.tab;
                            this.switchTab(tabName);
                        });
                    });

                    // Blob storage events
                    if (this.dom.refreshBlobsBtn) {
                        this.dom.refreshBlobsBtn.addEventListener('click', () => this.loadBlobFiles());
                    }

                    // Cluster events
                    if (this.dom.startAllBtn) {
                        this.dom.startAllBtn.addEventListener('click', () => this.manageCluster('start'));
                        this.dom.stopAllBtn.addEventListener('click', () => this.manageCluster('stop'));
                        this.dom.restartAllBtn.addEventListener('click', () => this.manageCluster('restart'));
                        this.dom.refreshClusterBtn.addEventListener('click', () => this.loadClusterStatus());
                    }

                    this.dom.refreshKeysBtn.addEventListener('click', () => this.loadKeys());
                    this.dom.addKeyBtn.addEventListener('click', () => this.showAddKeyModal());
                    this.dom.saveBtn.addEventListener('click', () => this.saveValue());
                    this.dom.deleteBtn.addEventListener('click', () => this.confirmDeleteKey());
                    this.dom.formatBtn.addEventListener('click', () => this.formatJson());
                    this.dom.keySearchInput.addEventListener('input', (e) => this.renderKeyTree(e.target.value));
                    this.dom.modeSelect.addEventListener('change', (e) => this.changeMode(e.target.value));

                    this.dom.keyTreeContainer.addEventListener('click', (e) => {
                        const label = e.target.closest('.node-label');
                        if (!label) return;
                        const node = label.parentElement;
                        if (node.classList.contains('tree-folder')) {
                            node.classList.toggle('open');
                        } else if (node.dataset.key) {
                            this.selectKey(node.dataset.key);
                        }
                    });
                }

                async apiRequest(endpoint, payload = null, method = 'POST') {
                    if (!window.TB?.api?.request) {
                        console.error("TB.api not available!");
                        return { error: true, message: "TB.api not available" };
                    }
                    try {
                        const url = (method === 'GET' && payload) ? `${endpoint}?${new URLSearchParams(payload)}` : endpoint;
                        const body = (method !== 'GET') ? payload : null;
                        const response = await window.TB.api.request(API_NAME, url, body, method);

                        if (response.error && response.error !== 'none') {
                            const errorMsg = response.info?.help_text || response.error;
                            console.error(`API Error on ${endpoint}:`, errorMsg, response);
                            if (window.TB?.ui?.Toast) TB.ui.Toast.showError(errorMsg, { duration: 5000 });
                            return { error: true, message: errorMsg, data: response.get() };
                        }
                        return { error: false, data: response.get() };
                    } catch (err) {
                        console.error("Framework/Network Error:", err);
                        if (window.TB?.ui?.Toast) TB.ui.Toast.showError("Application or network error.", { duration: 5000 });
                        return { error: true, message: "Network error" };
                    }
                }

                async loadInitialStatus() {
                    const res = await this.apiRequest('api_get_status', null, 'GET');
                    if (!res.error) this.dom.modeSelect.value = res.data.mode;
                }

                async loadKeys() {
                    this.setStatusMessage('Loading keys...');
                    const res = await this.apiRequest('api_get_all_keys', null, 'GET');
                    if (!res.error) {
                        this.cache.keys = res.data || [];
                        this.renderKeyTree();
                    } else {
                        this.setStatusMessage('Failed to load keys.', true);
                    }
                }

                renderKeyTree(filter = '') {
                    const treeData = {};
                    const filteredKeys = this.cache.keys.filter(k => k.toLowerCase().includes(filter.toLowerCase().trim()));

                    for (const key of filteredKeys) {
                        let currentLevel = treeData;
                        const parts = key.split(':');
                        for (let i = 0; i < parts.length; i++) {
                            const part = parts[i];
                            if (!part) continue; // Skip empty parts from keys like "a::b"
                            const isLeaf = i === parts.length - 1;

                            if (!currentLevel[part]) {
                                currentLevel[part] = { _children: {} };
                            }
                            if (isLeaf) {
                                currentLevel[part]._fullKey = key;
                            }
                            currentLevel = currentLevel[part]._children;
                        }
                    }

                    const treeHtml = this.buildTreeHtml(treeData);
                    if (treeHtml) {
                        this.dom.keyTreeContainer.innerHTML = `<ul class="key-tree">${treeHtml}</ul>`;
                        // Re-select the key if it's still visible
                        if (this.cache.selectedKey) {
                             const nodeEl = this.dom.keyTreeContainer.querySelector(`[data-key="${this.cache.selectedKey}"] .node-label`);
                             if(nodeEl) nodeEl.classList.add('selected');
                        }
                    } else {
                         this.setStatusMessage(filter ? 'No keys match your search.' : 'No keys found.');
                    }
                }

                buildTreeHtml(node) {
                    return Object.keys(node).sort().map(key => {
                        const childNode = node[key];
                        const isFolder = Object.keys(childNode._children).length > 0;

                        if (isFolder) {
                            return `<li class="tree-folder" ${childNode._fullKey ? `data-key="${childNode._fullKey}"`: ''}>
                                        <div class="node-label"><i class="node-icon"></i>${key}</div>
                                        <ul class="tree-children">${this.buildTreeHtml(childNode._children)}</ul>
                                    </li>`;
                        } else {
                            return `<li class="tree-leaf" data-key="${childNode._fullKey}">
                                        <div class="node-label"><i class="node-icon"></i>${key}</div>
                                    </li>`;
                        }
                    }).join('');
                }

                async selectKey(key) {
                    if (!key) return;
                    this.showEditor(true);
                    this.cache.selectedKey = key;

                    document.querySelectorAll('.node-label.selected').forEach(el => el.classList.remove('selected'));
                    const nodeEl = this.dom.keyTreeContainer.querySelector(`[data-key="${key}"] > .node-label`);
                    if (nodeEl) nodeEl.classList.add('selected');

                    this.dom.selectedKey.textContent = key;
                    this.dom.selectedKey.title = key;
                    this.dom.valueEditor.value = "Loading...";

                    const res = await this.apiRequest('api_get_value', { key }, 'GET');
                    this.dom.valueEditor.value = res.error ? `Error: ${res.message}` : res.data.value;
                    if (!res.error) this.formatJson(false); // Auto-format if it's valid JSON, without showing an error
                }

                async saveValue() {
                    if (!this.cache.selectedKey) return;
                    if (window.TB?.ui?.Loader) TB.ui.Loader.show("Saving...");
                    const res = await this.apiRequest('api_set_value', {
                        key: this.cache.selectedKey,
                        value: this.dom.valueEditor.value
                    });
                    if (window.TB?.ui?.Loader) TB.ui.Loader.hide();
                    if (!res.error && window.TB?.ui?.Toast) TB.ui.Toast.showSuccess("Key saved successfully!");
                }

                async confirmDeleteKey() {
                    if (!this.cache.selectedKey) return;
                    if (!window.TB?.ui?.Modal) {
                        if(confirm(`Delete key "${this.cache.selectedKey}"?`)) this.deleteKey();
                        return;
                    }
                    TB.ui.Modal.confirm({
                        title: 'Delete Key?',
                        content: `Are you sure you want to delete the key "<strong>${this.cache.selectedKey}</strong>"?<br/>This action cannot be undone.`,
                        confirmButtonText: 'Delete',
                        confirmButtonVariant: 'danger',
                        onConfirm: () => this.deleteKey()
                    });
                }

                async deleteKey() {
                    const keyToDelete = this.cache.selectedKey;
                    if (!keyToDelete) return;
                    if (window.TB?.ui?.Loader) TB.ui.Loader.show("Deleting...");
                    const res = await this.apiRequest('api_delete_key', { key: keyToDelete });
                    if (window.TB?.ui?.Loader) TB.ui.Loader.hide();

                    if (!res.error) {
                        if (window.TB?.ui?.Toast) TB.ui.Toast.showSuccess(`Key "${keyToDelete}" deleted.`);
                        this.cache.selectedKey = null;
                        this.showEditor(false);
                        this.loadKeys(); // Refresh the key list
                    }
                }

                formatJson(showErrorToast = true) {
                    try {
                        const currentVal = this.dom.valueEditor.value.trim();
                        if (!currentVal) return;
                        const formatted = JSON.stringify(JSON.parse(currentVal), null, 2);
                        this.dom.valueEditor.value = formatted;
                    } catch (e) {
                        if (showErrorToast && window.TB?.ui?.Toast) {
                            TB.ui.Toast.showWarning("Value is not valid JSON.", { duration: 3000 });
                        }
                    }
                }

                showAddKeyModal() {
                     if (!window.TB?.ui?.Modal) { alert("Add Key modal not available."); return; }
                     TB.ui.Modal.show({
                        title: 'Add New Key',
                        content: `<input type="text" id="newKeyInput" placeholder="Enter new key name (e.g., app:settings:user)" style="width: 100%; margin-bottom: 1rem;"/>
                                  <textarea id="newValueInput" placeholder='Enter value (e.g., {"theme": "dark"})' style="width: 100%; height: 150px; font-family: var(--font-family-mono);"></textarea>`,
                        onOpen: (modal) => document.getElementById('newKeyInput').focus(),
                        buttons: [{
                            text: 'Save', variant: 'primary',
                            action: async (modal) => {
                                const newKey = document.getElementById('newKeyInput').value.trim();
                                const newValue = document.getElementById('newValueInput').value;
                                if (!newKey) { if (window.TB?.ui?.Toast) TB.ui.Toast.showError("Key name cannot be empty."); return; }
                                modal.close();
                                if (window.TB?.ui.Loader) TB.ui.Loader.show("Saving...");
                                const res = await this.apiRequest('api_set_value', { key: newKey, value: newValue });
                                if (window.TB?.ui.Loader) TB.ui.Loader.hide();
                                if (!res.error) {
                                    if (window.TB?.ui?.Toast) TB.ui.Toast.showSuccess("New key created!");
                                    await this.loadKeys();
                                    this.selectKey(newKey);
                                }
                            }
                        }, { text: 'Cancel', action: (modal) => modal.close() }]
                    });
                }

                async changeMode(newMode) {
                    if (window.TB?.ui?.Loader) TB.ui.Loader.show(`Switching to ${newMode}...`);
                    const res = await this.apiRequest('api_change_mode', { mode: newMode });
                    if (!res.error) {
                       this.cache.selectedKey = null;
                       this.showEditor(false);
                       await this.loadKeys();
                       if (window.TB?.ui?.Toast) TB.ui.Toast.showSuccess(`Switched to ${newMode} mode.`);
                    } else {
                       if (window.TB?.ui?.Toast) TB.ui.Toast.showError(`Failed to switch mode.`);
                       await this.loadInitialStatus(); // Revert dropdown to actual status
                    }
                    if (window.TB?.ui?.Loader) TB.ui.Loader.hide();
                }

                showEditor(show) {
                    this.dom.editorPanel.classList.toggle('hidden', !show);
                    this.dom.placeholderPanel.classList.toggle('hidden', show);
                }

                setStatusMessage(message, isError = false) {
                    this.dom.keyTreeContainer.innerHTML = `<p class="status-message" style="${isError ? 'color: var(--color-danger);' : ''}">${message}</p>`;
                }

                switchTab(tabName) {
                    // Update tab buttons
                    this.dom.tabButtons.forEach(btn => btn.classList.remove('active'));
                    document.querySelector(`[data-tab="${tabName}"]`).classList.add('active');

                    // Update tab content
                    this.dom.tabContents.forEach(content => content.classList.remove('active'));
                    document.getElementById(`${tabName}-tab`).classList.add('active');

                    // Load tab-specific data
                    if (tabName === 'blob-storage' && isAdminUser) {
                        this.loadBlobStatus();
                        this.loadBlobFiles();
                    } else if (tabName === 'cluster' && isAdminUser) {
                        this.loadClusterStatus();
                    }
                }

                async loadBlobStatus() {
                    const res = await this.apiRequest('api_get_blob_status', null, 'GET');
                    if (!res.error) {
                        const data = res.data;
                        this.dom.blobStorageStatus.innerHTML = `
                            <div><span class="status-indicator status-${data.status === 'available' ? 'online' : 'offline'}"></span>${data.status}</div>
                            <div>Storage: ${data.storage_dir}</div>
                        `;

                        const serverHtml = data.servers.map(server =>
                            `<div><span class="status-indicator status-${server.status}"></span>${server.address}</div>`
                        ).join('');
                        this.dom.serverHealth.innerHTML = serverHtml || 'No servers';
                    }
                }

                async loadBlobFiles() {
                    const res = await this.apiRequest('api_list_blob_files', null, 'GET');
                    if (!res.error) {
                        this.cache.blobFiles = res.data;
                        this.renderBlobFiles();
                    }
                }

                renderBlobFiles() {
                    if (this.cache.blobFiles.length === 0) {
                        this.dom.blobFileList.innerHTML = '<div class="blob-file-item">No blob files found</div>';
                        return;
                    }

                    const html = this.cache.blobFiles.map(file => `
                        <div class="blob-file-item">
                            <div>
                                <strong>${file.id}</strong>
                                <div style="font-size: 0.875rem; color: var(--color-text-muted);">
                                    ${this.formatBytes(file.size)} • ${file.created} • ${file.encrypted ? '🔒 Encrypted' : '🔓 Plain'}
                                </div>
                            </div>
                        </div>
                    `).join('');

                    this.dom.blobFileList.innerHTML = html;
                }

                async loadClusterStatus() {
                    const res = await this.apiRequest('api_get_cluster_status', null, 'GET');
                    if (!res.error) {
                        this.cache.clusterStatus = res.data;
                        this.renderClusterStatus();
                    }
                }

                renderClusterStatus() {
                    if (!this.cache.clusterStatus) return;

                    const html = this.cache.clusterStatus.instances.map(instance => `
                        <div class="instance-item">
                            <div>
                                <strong>${instance.id}</strong>
                                <div style="font-size: 0.875rem; color: var(--color-text-muted);">
                                    ${instance.host}:${instance.port} • PID: ${instance.pid || 'N/A'} • ${instance.version || 'Unknown'}
                                </div>
                            </div>
                            <div>
                                <span class="status-indicator status-${instance.status}"></span>
                                ${instance.status}
                            </div>
                        </div>
                    `).join('');

                    this.dom.instanceList.innerHTML = html;
                }

                async manageCluster(action) {
                    if (window.TB?.ui?.Loader) TB.ui.Loader.show(`${action}ing cluster...`);
                    const res = await this.apiRequest('api_manage_cluster', { action });
                    if (window.TB?.ui?.Loader) TB.ui.Loader.hide();

                    if (!res.error) {
                        if (window.TB?.ui?.Toast) TB.ui.Toast.showSuccess(`Cluster ${action} completed`);
                        setTimeout(() => this.loadClusterStatus(), 2000);
                    }
                }

                formatBytes(bytes) {
                    if (bytes === 0) return '0 Bytes';
                    const k = 1024;
                    const sizes = ['Bytes', 'KB', 'MB', 'GB'];
                    const i = Math.floor(Math.log(bytes) / Math.log(k));
                    return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
                }
            }

            function onTbReady() { new DBManager(); }
            if (window.TB?.events) {
                if (window.TB.config?.get('appRootId')) {
                    onTbReady();
                } else {
                    window.TB.events.on('tbjs:initialized', onTbReady, { once: true });
                }
            } else {
                document.addEventListener('tbjs:initialized', onTbReady, { once: true });
            }
        </script>
    </body>
    </html>
    """
    app = get_app(Name)
    try:
        # Prepend the web context to include necessary framework scripts (like TB.js)
        web_context = app.web_context()
        return Result.html(web_context + html_content)
    except Exception:
        # Fallback in case web_context is not available
        return Result.html(html_content)

EventManager

module

EventManagerClass

Source code in toolboxv2/mods/EventManager/module.py

class EventManagerClass:
    events: set[Event] = set()
    source_id: str
    _name: str
    _identification: str

    routes_client: dict[str, ProxyRout] = {}
    routers_servers: dict[str, DaemonRout] = {}
    routers_servers_tasks: list[Any] = []
    routers_servers_tasks_running_flag: bool = False

    receiver_que: queue.Queue
    response_que: queue.Queue

    def add_c_route(self, name, route: ProxyRout):
        self.routes_client[name] = route

    async def receive_all_client_data(self):

        close_connections = []
        add_ev = []
        for name, client in self.routes_client.items():
            if client.client is None or not client.client.get('alive', False):
                close_connections.append(name)
                continue
            data = client.r

            if isinstance(data, str) and data == "No data":
                continue
            elif isinstance(data, EventID) and len(data.get_source()) != 0:
                await self.trigger_event(data)
            elif isinstance(data, EventID) and len(data.get_source()) == 0:
                print(f"Event returned {data.payload}")
                self.response_que.put(data)
            elif isinstance(data,
                            dict) and 'error' in data and 'origin' in data and 'result' in data and 'info' in data:

                self.response_que.put(Result.result_from_dict(**data).print())
            elif isinstance(data,
                            dict) and 'source' in data and 'path' in data and 'ID' in data and 'identifier' in data:
                del data['identifier']
                ev_id = EventID(**data)
                await self.trigger_event(ev_id)
            elif isinstance(data, Event):
                print("Event:", str(data.event_id), data.name)
                add_ev.append(data)
            elif isinstance(data, Result):
                self.response_que.put(data.print())
            else:
                print(f"Unknown Data {data}")

        for ev in add_ev:
            await self.register_event(ev)

        for client_name in close_connections:
            print(f"Client {client_name} closing connection")
            self.remove_c_route(client_name)

    def remove_c_route(self, name):
        self.routes_client[name].close()
        del self.routes_client[name]

    def crate_rout(self, source, addr=None):
        if addr is None:
            addr = ('0.0.0.0', 6588)
        host, port = addr
        if isinstance(port, str):
            port = int(port)
        return Rout(
            _from=self.source_id,
            _to=source,
            _from_port=int(os.getenv("TOOLBOXV2_BASE_PORT", 6588)),
            _from_host=os.getenv("TOOLBOXV2_BASE_HOST"),
            _to_port=port,
            _to_host=host,
            routing_function=self.routing_function_router,
        )

    def __init__(self, source_id, _identification="PN"):
        self.bo = False
        self.running = False
        self.source_id = source_id
        self.receiver_que = queue.Queue()
        self.response_que = queue.Queue()
        self._identification = _identification
        self._name = self._identification + '-' + str(uuid.uuid4()).split('-')[1]
        self.routes = {}
        self.logger = get_logger()

    @property
    def identification(self) -> str:
        return self._identification

    @identification.setter
    def identification(self, _identification: str):
        self.stop()
        self._identification = _identification
        self._name = self._identification + '-' + str(uuid.uuid4()).split('-')[1]

    async def identity_post_setter(self):

        do_reconnect = len(list(self.routers_servers.keys())) > 0
        if self._identification == "P0":
            await self.add_server_route(self._identification, ('0.0.0.0', 6568))
        if self._identification == "P0|S0":
            await self.add_server_route(self._identification, ('0.0.0.0', 6567))

        await asyncio.sleep(0.1)
        self.start()
        await asyncio.sleep(0.1)
        if do_reconnect:
            self.reconnect("ALL")

    async def open_connection_server(self, port):
        await self.add_server_route(self._identification, ('0.0.0.0', port))

    def start(self):
        self.running = True
        threading.Thread(target=async_test(self.receiver), daemon=True).start()

    def make_event_from_fuction(self, fuction, name, *args, source_types=SourceTypes.F,
                                scope=Scope.local,
                                exec_in=ExecIn.local,
                                threaded=False, **kwargs):

        return Event(source=fuction,
                     name=name,
                     event_id=EventID.crate_with_source(self.source_id), args=args,
                     kwargs_=kwargs,
                     source_types=source_types,
                     scope=scope,
                     exec_in=exec_in,
                     threaded=threaded,
                     )

    async def add_client_route(self, source_id, addr):
        if source_id in self.routes_client:
            if self.routes_client[source_id].client is None or not self.routes_client[source_id].client.get('alive'):
                await self.routes_client[source_id].reconnect()
                return True
            print("Already connected")
            return False
        try:
            pr = await ProxyRout.toProxy(rout=self.crate_rout(source_id, addr=addr), name=source_id)
            await asyncio.sleep(0.1)
            await pr.client.get('sender')({"id": self._identification,
                                           "continue": False,
                                           "key": os.getenv('TB_R_KEY', 'root@remote')})
            await asyncio.sleep(0.1)
            self.add_c_route(source_id, pr)
            return True
        except Exception as e:
            print(f"Check the port {addr} Sever likely not Online : {e}")
            return False

    async def add_mini_client(self, name: str, addr: tuple[str, int]):

        mini_proxy = await ProxyRout(class_instance=None, timeout=15, app=get_app(),
                                     remote_functions=[""], peer=False, name=name, do_connect=False)

        async def _(x):
            return await self.routers_servers[self._identification].send(x, addr)

        mini_proxy.put_data = _
        mini_proxy.connect = lambda *x, **_: None
        mini_proxy.reconnect = lambda *x, **_: None
        mini_proxy.close = lambda *x, **_: None
        mini_proxy.client = {'alive': True}
        mini_proxy.r = "No data"
        self.routes_client[name] = mini_proxy

    async def on_register(self, id_, data):
        try:
            if "unknown" not in self.routes:
                self.routes["unknown"] = {}

            if id_ != "new_con" and 'id' in data:
                id_data = data.get('id')
                id_ = eval(id_)
                c_host, c_pot = id_
                print(f"Registering: new client {id_data} : {c_host, c_pot}")
                if id_data not in self.routes_client:
                    await self.add_mini_client(id_data, (c_host, c_pot))
                    self.routes[str((c_host, c_pot))] = id_data

            # print("self.routes:", self.routes)
        except Exception as e:
            print("Error in on_register", str(e))

    def on_client_exit(self, id_):

        if isinstance(id_, str):
            id_ = eval(id_)

        c_name = self.routes.get(id_)

        if c_name is None:
            return

        if c_name in self.routes_client:
            self.remove_c_route(c_name)
            print(f"Removed route to {c_name}")

    async def add_server_route(self, source_id, addr=None):
        if addr is None:
            addr = ('0.0.0.0', 6588)
        try:
            self.routers_servers[source_id] = await DaemonRout(rout=self.crate_rout(source_id, addr=addr),
                                                               name=source_id,
                                                               on_r=self.on_register)
            self.routers_servers_tasks.append(self.routers_servers[source_id].online)
        except Exception as e:
            print(f"Sever already Online : {e}")

        if not self.routers_servers_tasks_running_flag:
            self.routers_servers_tasks_running_flag = True
            threading.Thread(target=self.server_route_runner, daemon=True).start()

    def server_route_runner(self):
        loop = asyncio.new_event_loop()
        asyncio.set_event_loop(loop)

        # Sammle alle Ergebnisse zusammen
        results = loop.run_until_complete(asyncio.gather(*self.routers_servers_tasks))

        for result in results:
            print(result)

        loop.close()
        self.routers_servers_tasks_running_flag = False

    async def add_js_route(self, source_id="js:web"):
        await self.add_server_route(source_id, ("./web/scripts/tb_socket.sock", 0))

    async def register_event(self, event: Event):

        if event in self.events:
            return Result.default_user_error("Event registration failed Event already registered")

        print(f"Registration new Event : {event.name}, {str(event.event_id)}")
        self.events.add(event)

        if event.scope.name == Scope.instance.name:
            return

        if event.scope.name == Scope.local.name:
            if not self.bo and "P0" not in self.routes_client and os.getenv("TOOLBOXV2_BASE_HOST",
                                                                            "localhost") != "localhost":
                await self.add_client_route("P0", (os.getenv("TOOLBOXV2_BASE_HOST", "localhost"),
                                                   os.getenv("TOOLBOXV2_BASE_PORT", 6568)))
                self.bo = True
            return

        if event.scope.name == Scope.local_network.name:
            if self.identification == "P0" and not self.bo:
                t0 = threading.Thread(target=self.start_brodcast_router_local_network, daemon=True)
                t0.start()
            elif not self.bo and "P0" not in self.routes_client and os.getenv("TOOLBOXV2_BASE_HOST",
                                                                              "localhost") == "localhost":
                self.bo = True
                # self.add_server_route(self.identification, ("127.0.0.1", 44667))
                with Spinner(message="Sercheing for Rooter instance", count_down=True, time_in_s=6):
                    with ThreadPoolExecutor(max_workers=1) as executor:
                        t0 = executor.submit(make_known, self.identification)
                        try:
                            data = t0.result(timeout=6)
                        except TimeoutError:
                            print("No P0 found in network or on device")
                            return
                    print(f"Found P0 on {type(data)} {data.get('host')}")
                    await self.add_client_route("P0", (data.get("host"), os.getenv("TOOLBOXV2_BASE_PORT", 6568)))
            elif not self.bo and "P0" not in self.routes_client and os.getenv("TOOLBOXV2_BASE_HOST",
                                                                              "localhost") != "localhost":
                do = await self.add_client_route("P0", (
                    os.getenv("TOOLBOXV2_BASE_HOST", "localhost"), os.getenv("TOOLBOXV2_BASE_PORT", 6568)))
                self.bo = do
                if not do:
                    print("Connection failed")
                    os.environ["TOOLBOXV2_BASE_HOST"] = "localhost"

        if event.scope.name == Scope.global_network.name:
            await self.add_server_route(self.source_id, ('0.0.0.0', os.getenv("TOOLBOXV2_REMOTE_PORT", 6587)))

    async def connect_to_remote(self, host=os.getenv("TOOLBOXV2_REMOTE_IP"),
                                port=os.getenv("TOOLBOXV2_REMOTE_PORT", 6587)):
        await self.add_client_route("S0", (host, port))

    def start_brodcast_router_local_network(self):
        self.bo = True

        # print("Starting brodcast router 0")
        router = start_client(get_local_ip())
        # print("Starting brodcast router 1")
        # next(router)
        # print("Starting brodcast router")
        while self.running:
            source_id, connection = next(router)
            print(f"Infos :{source_id}, connection :{connection}")
            self.routes[source_id] = connection[0]
            router.send(self.running)

        router.send("e")
        router.close()

    def _get_event_by_id_or_name(self, event_id: str or EventID):
        if isinstance(event_id, str):
            events = [e for e in self.events if e.name == event_id]
            if len(events) < 1:
                return Result.default_user_error("Event not registered")
            event = events[0]

        elif isinstance(event_id, EventID):
            events = [e for e in self.events if e.event_id.ID == event_id.ID]
            if len(events) < 1:
                events = [e for e in self.events if e.name == event_id.ID]
            if len(events) < 1:
                return Result.default_user_error("Event not registered")
            event = events[0]

        elif isinstance(event_id, Event):
            if event_id not in self.events:
                return Result.default_user_error("Event not registered")
            event = event_id

        else:
            event = Result.default_user_error("Event not registered")

        return event

    def remove_event(self, event: Event or EventID or str):

        event = self._get_event_by_id_or_name(event)
        if isinstance(event, Event):
            self.events.remove(event)
        else:
            return event

    async def _trigger_local(self, event_id: EventID):
        """
        Exec source based on

        source_types
            F -> call directly
            R -> use get_app(str(event_id)).run_any(*args, **kwargs)
            S -> evaluate string
        scope
            instance -> _trigger_local
            local -> if you ar proxy app run the event through get_app(str(event_id)).run_any(TBEF.EventManager._trigger_local, args=args, kwargs=kwargs, get_result=True)
            local_network -> use proxy0 app to communicate withe Daemon0 then local
            global_network ->
        exec_in
        event_id
        threaded

                       """
        event = self._get_event_by_id_or_name(event_id)

        if isinstance(event, Result):
            event.print()
            if self.identification == "P0":
                return event
            print(f"Routing to P0 {self.events}")
            if self.source_id not in self.routes_client:
                # self.routers[self.source_id] = DaemonRout(rout=self.crate_rout(self.source_id))
                await self.add_client_route("P0", ('127.0.0.1', 6568))
            return await self.route_event_id(event_id)

        # if event.threaded:
        #    threading.Thread(target=self.runner, args=(event, event_id), daemon=True).start()
        #    return "Event running In Thread"
        # else:

        return await self.runner(event, event_id)

    async def runner(self, event, event_id: EventID):

        if event.kwargs_ is None:
            event.kwargs_ = {}
        if event.args is None:
            event.args = []

        if event.source_types.name is SourceTypes.P.name:
            return event.source(*event.args, payload=event_id, **event.kwargs_)

        if event.source_types.name is SourceTypes.F.name:
            return event.source(*event.args, **event.kwargs_)

        if event.source_types.name is SourceTypes.R.name:
            return get_app(str(event_id)).run_any(mod_function_name=event.source, get_results=True, args_=event.args,
                                                  kwargs_=event.kwargs_)

        if event.source_types.name is SourceTypes.AP.name:
            if 'payload' in event.kwargs_:
                if event_id.payload != event.kwargs_['payload']:
                    event_id.payload = event.kwargs_['payload']
                del event.kwargs_['payload']
            print(event.args, event.kwargs_, "TODO: remove")
            return await event.source(*event.args, payload=event_id, **event.kwargs_)

        if event.source_types.name is SourceTypes.AF.name:
            return await event.source(*event.args, **event.kwargs_)

        if event.source_types.name is SourceTypes.AR.name:
            return await get_app(str(event_id)).run_any(mod_function_name=event.source, get_results=True,
                                                        args_=event.args,
                                                        kwargs_=event.kwargs_)

        if event.source_types.name is SourceTypes.S.name:
            return eval(event.source, __locals={'app': get_app(str(event_id)), 'event': event, 'eventManagerC': self})

    async def routing_function_router(self, event_id: EventID):

        result = await self.trigger_event(event_id)

        if result is None:
            result = Result.default_user_error("Invalid Event ID")

        if isinstance(result, bytes | dict):
            pass
        elif isinstance(result, Result):
            result.result.data_info = str(event_id)
        elif isinstance(result, EventID):
            result = Result.default_internal_error("Event not found", data=result)
        else:
            result = Result.ok(data=result, data_info="<automatic>", info=str(event_id.path))

        if isinstance(result, str):
            result = result.encode()

        return result

    async def trigger_evnet_by_name(self, name: str):
        await self.trigger_event(EventID.crate_name_as_id(name=name))

    async def trigger_event(self, event_id: EventID):
        """
        Exec source based on

        source_types
            F -> call directly
            R -> use get_app(str(event_id)).run_any(*args, **kwargs)
            S -> evaluate string
        scope
            instance -> _trigger_local
            local -> if you ar proxy app run the event through get_app(str(event_id)).run_any(TBEF.EventManager._trigger_local, args=args, kwargs=kwargs, get_result=True)
            local_network -> use proxy0 app to communicate withe Daemon0 then local
            global_network ->
        exec_in
        event_id
        threaded

                       """
        # print(f"event-id Ptah : {event_id.get_path()}")
        # print(f"testing trigger_event for {event_id.get_source()} {event_id.get_source()[-1] == self.source_id} ")
        print(str(event_id))
        if event_id.get_source()[-1] == self.source_id:
            payload = await self._trigger_local(event_id)
            event_id.set_payload(payload)
            if len(event_id.path) > 1:
                event_id.source = ':'.join([e.split(':')[0] for e in event_id.get_path() if e != "E"])
                res = await self.route_event_id(event_id)
                if isinstance(res, Result):
                    res.print()
                else:
                    print(res)
            return payload
        return await self.route_event_id(event_id)

    async def route_event_id(self, event_id: EventID):

        # print(f"testing route_event_id for {event_id.get_source()[-1]}")
        if event_id.get_source()[-1] == '*':  # self.identification == "P0" and
            responses = []
            event_id.source = ':'.join(event_id.get_source()[:-1])
            event_id.add_path(f"{self._name}({self.source_id})")
            data = asdict(event_id)
            for name, rout_ in self.routes_client.items():
                if name in event_id.path:
                    continue
                ret = await rout_.put_data(data)
                responses.append(ret)
            return responses
        route = self.routes_client.get(event_id.get_source()[-1])
        # print("route:", route)
        if route is None:
            route = self.routes_client.get(event_id.get_path()[-1])
        if route is None:
            return event_id.add_path(("" if len(event_id.get_source()) == 1 else "404#")+self.identification)
        time.sleep(0.25)
        event_id.source = ':'.join(event_id.get_source()[:-1])
        event_id.add_path(f"{self._name}({self.source_id})")
        return await route.put_data(asdict(event_id))

    async def receiver(self):

        t0 = time.time()

        while self.running:
            time.sleep(0.25)
            if not self.receiver_que.empty():
                event_id = self.receiver_que.get()
                print("Receiver Event", str(event_id))
                await self.trigger_event(event_id)

            if time.time() - t0 > 5:
                await self.receive_all_client_data()
                t0 = time.time()

    def info(self):
        return {"source": self.source_id, "known_routs:": self.routers_servers, "_router": self.routes_client,
                "events": self.events}

    def stop(self):
        self.running = False
        list(map(lambda x: x.disconnect(), self.routes_client.values()))
        list(map(lambda x: x.stop(), self.routers_servers.values()))

    def reconnect(self, name):
        if name is None:
            pass
        elif name in self.routes_client:
            self.routes_client[name].reconnect()
            return
        list(map(lambda x: x.reconnect(), self.routes_client.values()))

    async def verify(self, name):
        if name is None:
            pass
        elif name in self.routes_client:
            await self.routes_client[name].verify()
            return
        for x in self.routes_client.values():
            await x.verify()

trigger_event(event_id) async

Exec source based on

source_types F -> call directly R -> use get_app(str(event_id)).run_any(args, *kwargs) S -> evaluate string scope instance -> _trigger_local local -> if you ar proxy app run the event through get_app(str(event_id)).run_any(TBEF.EventManager._trigger_local, args=args, kwargs=kwargs, get_result=True) local_network -> use proxy0 app to communicate withe Daemon0 then local global_network -> exec_in event_id threaded

Source code in toolboxv2/mods/EventManager/module.py

async def trigger_event(self, event_id: EventID):
    """
    Exec source based on

    source_types
        F -> call directly
        R -> use get_app(str(event_id)).run_any(*args, **kwargs)
        S -> evaluate string
    scope
        instance -> _trigger_local
        local -> if you ar proxy app run the event through get_app(str(event_id)).run_any(TBEF.EventManager._trigger_local, args=args, kwargs=kwargs, get_result=True)
        local_network -> use proxy0 app to communicate withe Daemon0 then local
        global_network ->
    exec_in
    event_id
    threaded

                   """
    # print(f"event-id Ptah : {event_id.get_path()}")
    # print(f"testing trigger_event for {event_id.get_source()} {event_id.get_source()[-1] == self.source_id} ")
    print(str(event_id))
    if event_id.get_source()[-1] == self.source_id:
        payload = await self._trigger_local(event_id)
        event_id.set_payload(payload)
        if len(event_id.path) > 1:
            event_id.source = ':'.join([e.split(':')[0] for e in event_id.get_path() if e != "E"])
            res = await self.route_event_id(event_id)
            if isinstance(res, Result):
                res.print()
            else:
                print(res)
        return payload
    return await self.route_event_id(event_id)

Rout dataclass

Source code in toolboxv2/mods/EventManager/module.py

@dataclass
class Rout:
    _from: str
    _to: str

    _from_port: int
    _from_host: str

    _to_port: int
    _to_host: str

    routing_function: Callable

    @property
    def to_host(self):
        return self._to_host

    @property
    def to_port(self):
        return self._to_port

    async def put_data(self, event_id_data: dict[str, str]):
        event_id: EventID = EventID(**event_id_data)
        return await self.routing_function(event_id)

    def close(self):
        """ Close """

close()

Close

Source code in toolboxv2/mods/EventManager/module.py

def close(self):
    """ Close """

FileWidget

FileUploadHandler

Source code in toolboxv2/mods/FileWidget.py

class FileUploadHandler:
    def __init__(self, upload_dir: str = 'uploads'):
        self.upload_dir = Path(upload_dir)
        self.upload_dir.mkdir(parents=True, exist_ok=True)
        # self.app = get_app().app # If logger is needed here

    def save_file(self, chunk_info: ChunkInfo, storage: BlobStorage) -> str:
        """Speichert die Datei oder Chunk. Chunks werden lokal gespeichert, dann zu BlobStorage gemerged."""
        final_blob_path = Path(chunk_info.filename).name  # Use only filename part for security within blob storage

        if chunk_info.total_chunks == 1:
            # Komplette Datei direkt in BlobStorage speichern
            # print(f"Saving single part file: {final_blob_path} to BlobStorage directly.") # Debug
            with BlobFile(final_blob_path, 'w', storage=storage) as bf:
                bf.write(chunk_info.content)
        else:
            # Chunk lokal speichern
            # Sanitize filename for local path (original chunk_info.filename might contain path parts client-side)
            safe_base_filename = "".join(
                c if c.isalnum() or c in ('.', '_', '-') else '_' for c in Path(chunk_info.filename).name)
            chunk_path = self.upload_dir / f"{safe_base_filename}.part{chunk_info.chunk_index}"
            # print(f"Saving chunk: {chunk_path} locally. Total chunks: {chunk_info.total_chunks}") # Debug

            with open(chunk_path, 'wb') as f:
                f.write(chunk_info.content)

            if self._all_chunks_received(safe_base_filename, chunk_info.total_chunks):
                # print(f"All chunks received for {safe_base_filename}. Merging to BlobStorage path: {final_blob_path}") # Debug
                self._merge_chunks_to_blob(safe_base_filename, chunk_info.total_chunks, final_blob_path, storage)
                self._cleanup_chunks(safe_base_filename, chunk_info.total_chunks)
            # else:
            # print(f"Still waiting for more chunks for {safe_base_filename}.") # Debug

        return final_blob_path  # Path within BlobStorage

    def _all_chunks_received(self, safe_base_filename: str, total_chunks: int) -> bool:
        for i in range(total_chunks):
            chunk_path = self.upload_dir / f"{safe_base_filename}.part{i}"
            if not chunk_path.exists():
                # print(f"Chunk {i} for {safe_base_filename} not found. Path: {chunk_path}") # Debug
                return False
        # print(f"All {total_chunks} chunks found for {safe_base_filename}.") # Debug
        return True

    def _merge_chunks_to_blob(self, safe_base_filename: str, total_chunks: int, final_blob_path: str,
                              storage: BlobStorage):
        # print(f"Merging {total_chunks} chunks for {safe_base_filename} into Blob: {final_blob_path}") # Debug
        with BlobFile(final_blob_path, 'w', storage=storage) as outfile:
            for i in range(total_chunks):
                chunk_path = self.upload_dir / f"{safe_base_filename}.part{i}"
                # print(f"Appending chunk {i} ({chunk_path}) to Blob.") # Debug
                with open(chunk_path, 'rb') as chunk_file:
                    outfile.write(chunk_file.read())
        # print(f"Finished merging chunks for {safe_base_filename} to Blob: {final_blob_path}") # Debug

    def _cleanup_chunks(self, safe_base_filename: str, total_chunks: int):
        # print(f"Cleaning up {total_chunks} chunks for {safe_base_filename}.") # Debug
        for i in range(total_chunks):
            chunk_path = self.upload_dir / f"{safe_base_filename}.part{i}"
            if chunk_path.exists():
                # print(f"Removing chunk: {chunk_path}") # Debug
                try:
                    os.remove(chunk_path)
                except OSError as e:
                    # self.app.logger.error(f"Error removing chunk {chunk_path}: {e}") # If logger available
                    print(f"Error removing chunk {chunk_path}: {e}")

save_file(chunk_info, storage)

Speichert die Datei oder Chunk. Chunks werden lokal gespeichert, dann zu BlobStorage gemerged.

Source code in toolboxv2/mods/FileWidget.py

def save_file(self, chunk_info: ChunkInfo, storage: BlobStorage) -> str:
    """Speichert die Datei oder Chunk. Chunks werden lokal gespeichert, dann zu BlobStorage gemerged."""
    final_blob_path = Path(chunk_info.filename).name  # Use only filename part for security within blob storage

    if chunk_info.total_chunks == 1:
        # Komplette Datei direkt in BlobStorage speichern
        # print(f"Saving single part file: {final_blob_path} to BlobStorage directly.") # Debug
        with BlobFile(final_blob_path, 'w', storage=storage) as bf:
            bf.write(chunk_info.content)
    else:
        # Chunk lokal speichern
        # Sanitize filename for local path (original chunk_info.filename might contain path parts client-side)
        safe_base_filename = "".join(
            c if c.isalnum() or c in ('.', '_', '-') else '_' for c in Path(chunk_info.filename).name)
        chunk_path = self.upload_dir / f"{safe_base_filename}.part{chunk_info.chunk_index}"
        # print(f"Saving chunk: {chunk_path} locally. Total chunks: {chunk_info.total_chunks}") # Debug

        with open(chunk_path, 'wb') as f:
            f.write(chunk_info.content)

        if self._all_chunks_received(safe_base_filename, chunk_info.total_chunks):
            # print(f"All chunks received for {safe_base_filename}. Merging to BlobStorage path: {final_blob_path}") # Debug
            self._merge_chunks_to_blob(safe_base_filename, chunk_info.total_chunks, final_blob_path, storage)
            self._cleanup_chunks(safe_base_filename, chunk_info.total_chunks)
        # else:
        # print(f"Still waiting for more chunks for {safe_base_filename}.") # Debug

    return final_blob_path  # Path within BlobStorage

access_shared_file(self, request, share_id, filename=None, row=None) async

Accesses a shared file via its share_id. The URL for this would be like /api/FileWidget/shared/{share_id_value} The 'share_id: str' in signature implies ToolBoxV2 extracts it from path.

Source code in toolboxv2/mods/FileWidget.py

@export(mod_name=MOD_NAME, api=True, version=VERSION, name="open_shared", api_methods=['GET'],
        request_as_kwarg=True, level=-1, row=True)
async def access_shared_file(self, request: RequestData, share_id: str, filename: str = None, row=None) -> Result:  # share_id from query params
    """
    Accesses a shared file via its share_id.
    The URL for this would be like /api/FileWidget/shared/{share_id_value}
    The 'share_id: str' in signature implies ToolBoxV2 extracts it from path.
    """
    if not share_id:
        return Result.html(data="Share ID is missing in path.", status=302)

    share_info = self.shares.get(share_id) if self.shares is not None else None
    if not share_info:
        return Result.html(data="Share link is invalid or has expired.", status=404)

    owner_uid = share_info["owner_uid"]
    file_path_in_owner_storage = share_info["file_path"]

    try:
        # Get BlobStorage for the owner, not the current request's user (if any)
        owner_storage = await self.get_blob_storage(
            owner_uid_override=owner_uid)  # Crucially, pass request=None if not needed
        self.app.logger.info(
            f"Accessing shared file via link {share_id}: owner {owner_uid}, path {file_path_in_owner_storage}")
        result = await _prepare_file_response(self, owner_storage, file_path_in_owner_storage, row=row is not None)
        if result.is_error():
            self.app.logger.error(f"Error preparing shared file response for {share_id}: {result.info.help_text}")
            return Result.html(data=f"Failed to prepare shared file for download. {result.info.help_text} {result.result.data_info}")
        return result
    except ValueError as e:  # From get_blob_storage if owner_uid is invalid for some reason
        self.app.logger.error(f"Error getting owner's storage for shared file {share_id} (owner {owner_uid}): {e}",
                              exc_info=True)
        return Result.html(data="Could not access owner's storage for shared file.")
    except Exception as e:
        self.app.logger.error(
            f"Error accessing shared file {share_id} (owner {owner_uid}, path {file_path_in_owner_storage}): {e}",
            exc_info=True)
        return Result.html(data="Could not retrieve shared file.")

get_main_ui(self) async

Serves the main HTML UI for the FileWidget.

Source code in toolboxv2/mods/FileWidget.py

@export(mod_name=MOD_NAME, api=True, version=VERSION, name="ui", api_methods=['GET'])
async def get_main_ui(self) -> Result:
    """Serves the main HTML UI for the FileWidget."""
    html_content = get_template_content()
    return Result.html(data=html_content)

handle_upload(self, request, form_data=None) async

Handles file uploads. Expects chunked data via form_data kwarg from Rust server. 'form_data' structure (from Rust's parsing of multipart) after client sends FormData with fields: 'file' (the blob), 'fileName', 'chunkIndex', 'totalChunks'.

Expected form_data in this Python function: { "file": { // This 'file' key is the NAME of the form field that held the file blob "filename": "original_file_name_for_this_chunk.txt", // from Content-Disposition of the 'file' field part "content_type": "mime/type_of_chunk", "content_base64": "BASE64_ENCODED_CHUNK_CONTENT" }, "fileName": "overall_final_filename.txt", // From a separate form field named 'fileName' "chunkIndex": "0", // From a separate form field named 'chunkIndex' "totalChunks": "5" // From a separate form field named 'totalChunks' }

Source code in toolboxv2/mods/FileWidget.py

@export(mod_name=MOD_NAME, api=True, version=VERSION, name="upload", api_methods=['POST'], request_as_kwarg=True)
async def handle_upload(self, request: RequestData, form_data: dict[str, Any] | None = None) -> Result:
    """
    Handles file uploads. Expects chunked data via form_data kwarg from Rust server.
    'form_data' structure (from Rust's parsing of multipart) after client sends FormData with fields:
    'file' (the blob), 'fileName', 'chunkIndex', 'totalChunks'.

    Expected `form_data` in this Python function:
    {
        "file": {  // This 'file' key is the NAME of the form field that held the file blob
            "filename": "original_file_name_for_this_chunk.txt", // from Content-Disposition of the 'file' field part
            "content_type": "mime/type_of_chunk",
            "content_base64": "BASE64_ENCODED_CHUNK_CONTENT"
        },
        "fileName": "overall_final_filename.txt", // From a separate form field named 'fileName'
        "chunkIndex": "0",                        // From a separate form field named 'chunkIndex'
        "totalChunks": "5"                        // From a separate form field named 'totalChunks'
    }
    """
    self.app.logger.debug(
        f"FileWidget: handle_upload called. Received form_data keys: {list(form_data.keys()) if form_data else 'None'}"
    )
    self.app.logger.debug(f"FileWidget: handle_upload called. Received form_data: {request.to_dict()}")
    # self.app.logger.debug(f"Full form_data: {form_data}") # For deeper debugging if needed

    if not form_data:
        return Result.default_user_error(info="No form data received for upload.", exec_code=400)

    try:
        storage = await self.get_blob_storage(request)

        # Extract data from form_data (populated by Rust server from multipart)
        file_field_data = form_data.get('file')  # This is the dict from UploadedFile struct
        # The 'file_field_data.get('filename')' is the name of the chunk part,
        # which the JS client sets to be the same as the original file's name.
        # This is fine for FileUploadHandler.save_file's chunk_info.filename if total_chunks > 1,
        # as it will be used to create temporary part files like "original_file_name.txt.part0".

        overall_filename_from_form = form_data.get('fileName') # This is the target filename for the assembled file.
        chunk_index_str = form_data.get('chunkIndex')
        total_chunks_str = form_data.get('totalChunks')

        if not all([
            file_field_data, isinstance(file_field_data, dict),
            overall_filename_from_form,
            chunk_index_str is not None, # Check for presence, not just truthiness (0 is valid)
            total_chunks_str is not None # Check for presence
        ]):
            missing = []
            if not file_field_data or not isinstance(file_field_data, dict): missing.append("'file' object field")
            if not overall_filename_from_form: missing.append("'fileName' field")
            if chunk_index_str is None: missing.append("'chunkIndex' field")
            if total_chunks_str is None: missing.append("'totalChunks' field")

            self.app.logger.error(
                f"Missing critical form data fields for upload: {missing}. Received form_data: {form_data}")
            return Result.default_user_error(info=f"Incomplete upload data. Missing: {', '.join(missing)}",
                                             exec_code=400)

        content_base64 = file_field_data.get('content_base64')
        if not content_base64:
            return Result.default_user_error(info="File content (base64) not found in 'file' field data.",
                                             exec_code=400)

        try:
            content_bytes = base64.b64decode(content_base64)
        except base64.binascii.Error as b64_error:
            self.app.logger.error(f"Base64 decoding failed for upload: {b64_error}")
            return Result.default_user_error(info="Invalid file content encoding.", exec_code=400)

        try:
            chunk_index = int(chunk_index_str)
            total_chunks = int(total_chunks_str)
        except ValueError:
            return Result.default_user_error(info="Invalid chunk index or total chunks value. Must be integers.", exec_code=400)

        # Use the 'overall_filename_from_form' for the ChunkInfo.filename,
        # as this is the intended final name in blob storage.
        # FileUploadHandler will use Path(this_name).name to ensure it's just a filename.
        chunk_info_to_save = ChunkInfo(
            filename=overall_filename_from_form, # THIS IS THE KEY CHANGE FOR CONSISTENCY
            chunk_index=chunk_index,
            total_chunks=total_chunks,
            content=content_bytes
        )

        self.app.logger.info(
            f"Processing chunk {chunk_index + 1}/{total_chunks} for final file '{overall_filename_from_form}'. " # Log the intended final name
            f"Size: {len(content_bytes)} bytes."
        )

        saved_blob_path = self.upload_handler.save_file(chunk_info_to_save, storage) # saved_blob_path will be Path(overall_filename_from_form).name

        msg = f"Chunk {chunk_index + 1}/{total_chunks} for '{saved_blob_path}' saved."
        if chunk_info_to_save.chunk_index == chunk_info_to_save.total_chunks - 1:
            # Check if fully assembled
            # The 'safe_base_filename' in FileUploadHandler is derived from ChunkInfo.filename,
            # which we've now set to 'overall_filename_from_form'.
            # So, this check should work correctly.
            safe_base_filename_for_check = "".join(
                c if c.isalnum() or c in ('.', '_', '-') else '_' for c in Path(overall_filename_from_form).name)

            # A slight delay might be needed if file system operations are not instantly consistent across threads/processes
            # For now, assume direct check is okay.
            # await asyncio.sleep(0.1) # Optional small delay if race conditions are suspected with file system

            if self.upload_handler._all_chunks_received(safe_base_filename_for_check, total_chunks):
                msg = f"File '{saved_blob_path}' upload complete and assembled."
                self.app.logger.info(msg)
            else:
                msg = f"Final chunk for '{saved_blob_path}' saved, but assembly check failed or is pending."
                self.app.logger.warning(msg + f" (Could not verify all chunks for '{safe_base_filename_for_check}' immediately after final one)")


        return Result.ok(data={"message": msg, "path": saved_blob_path}) # Return the blob-relative path

    except ValueError as e:
        self.app.logger.error(f"Upload processing error: {e}", exc_info=True)
        return Result.default_user_error(info=f"Upload error: {str(e)}",
                                         exec_code=400 if "authentication" in str(e).lower() else 400)
    except Exception as e:
        self.app.logger.error(f"Unexpected error during file upload: {e}", exc_info=True)
        return Result.default_internal_error(info="An unexpected error occurred during upload.")

P2PRPCClient

P2PRPCClient

Source code in toolboxv2/mods/P2PRPCClient.py

class P2PRPCClient:
    def __init__(self, app: App, host: str, port: int, tb_r_key: str = None):
        self.app = app
        self.host = host
        self.port = port
        self.reader = None
        self.writer = None
        self.futures = {}
        self.code = Code()

        if tb_r_key is None:
            tb_r_key = os.getenv("TB_R_KEY")
            if tb_r_key is None:
                raise ValueError("TB_R_KEY environment variable is not set.")

        if len(tb_r_key) < 24:
            raise ValueError("TB_R_KEY must be at least 24 characters long for security.")
        self.auth_key_part = tb_r_key[:24]
        self.identification_part = tb_r_key[24:]
        self.session_key = None

    async def connect(self):
        """Connects to the local tcm instance and performs key exchange."""
        try:
            self.reader, self.writer = await asyncio.open_connection(self.host, self.port)
            print(f"RPC Client: Connected to tcm at {self.host}:{self.port}")

            # Receive encrypted session key from server
            len_data = await self.reader.readexactly(4)
            encrypted_session_key_len = int.from_bytes(len_data, 'big')
            encrypted_session_key = (await self.reader.readexactly(encrypted_session_key_len)).decode('utf-8')

            # Decrypt session key using auth_key_part
            self.session_key = self.code.decrypt_symmetric(encrypted_session_key, self.auth_key_part)

            # Send challenge back to server, encrypted with session key
            challenge = "CHALLENGE_ACK"
            encrypted_challenge = self.code.encrypt_symmetric(challenge, self.session_key)
            self.writer.write(len(encrypted_challenge).to_bytes(4, 'big'))
            self.writer.write(encrypted_challenge.encode('utf-8'))
            await self.writer.drain()

            # Start a background task to listen for responses
            asyncio.create_task(self.listen_for_responses())

        except ConnectionRefusedError:
            print(f"RPC Client: Connection to {self.host}:{self.port} refused. Is the tcm peer running?")
            raise
        except Exception as e:
            print(f"RPC Client: Error during connection/key exchange: {e}")
            raise

    async def listen_for_responses(self):
        """Listens for incoming responses, decrypts them, and resolves the corresponding future."""
        try:
            while True:
                len_data = await self.reader.readexactly(4)
                msg_len = int.from_bytes(len_data, 'big')
                encrypted_msg_data = (await self.reader.readexactly(msg_len)).decode('utf-8')

                decrypted_msg_data = self.code.decrypt_symmetric(encrypted_msg_data, self.session_key)
                response = json.loads(decrypted_msg_data)

                call_id = response.get('call_id')
                if call_id in self.futures:
                    future = self.futures.pop(call_id)
                    future.set_result(response)
        except asyncio.IncompleteReadError:
            print("RPC Client: Connection closed.")
        except Exception as e:
            print(f"RPC Client: Error listening for responses: {e}")
        finally:
            # Clean up any pending futures
            for future in self.futures.values():
                future.set_exception(ConnectionError("Connection lost"))
            self.futures.clear()

    async def call(self, module: str, function: str, *args, **kwargs):
        """Makes a remote procedure call."""
        if not self.writer:
            await self.connect()

        call_id = str(uuid.uuid4())
        request = {
            "type": "request",
            "call_id": call_id,
            "module": module,
            "function": function,
            "args": args,
            "kwargs": kwargs,
            "identification_part": self.identification_part
        }

        future = asyncio.get_running_loop().create_future()
        self.futures[call_id] = future

        try:
            request_str = json.dumps(request)
            encrypted_request = self.code.encrypt_symmetric(request_str, self.session_key)

            self.writer.write(len(encrypted_request).to_bytes(4, 'big'))
            self.writer.write(encrypted_request.encode('utf-8'))
            await self.writer.drain()

            # Wait for the response with a timeout
            response = await asyncio.wait_for(future, timeout=30.0)

            if response.get('error'):
                return Result(**response['error'])
            else:
                return Result.ok(response.get('result'))

        except TimeoutError:
            self.futures.pop(call_id, None)
            return Result.default_internal_error("RPC call timed out.")
        except Exception as e:
            self.futures.pop(call_id, None)
            return Result.default_internal_error(f"RPC call failed: {e}")

    async def close(self):
        """Closes the connection."""
        if self.writer:
            self.writer.close()
            await self.writer.wait_closed()
            print("RPC Client: Connection closed.")

call(module, function, *args, **kwargs) async

Makes a remote procedure call.

Source code in toolboxv2/mods/P2PRPCClient.py

async def call(self, module: str, function: str, *args, **kwargs):
    """Makes a remote procedure call."""
    if not self.writer:
        await self.connect()

    call_id = str(uuid.uuid4())
    request = {
        "type": "request",
        "call_id": call_id,
        "module": module,
        "function": function,
        "args": args,
        "kwargs": kwargs,
        "identification_part": self.identification_part
    }

    future = asyncio.get_running_loop().create_future()
    self.futures[call_id] = future

    try:
        request_str = json.dumps(request)
        encrypted_request = self.code.encrypt_symmetric(request_str, self.session_key)

        self.writer.write(len(encrypted_request).to_bytes(4, 'big'))
        self.writer.write(encrypted_request.encode('utf-8'))
        await self.writer.drain()

        # Wait for the response with a timeout
        response = await asyncio.wait_for(future, timeout=30.0)

        if response.get('error'):
            return Result(**response['error'])
        else:
            return Result.ok(response.get('result'))

    except TimeoutError:
        self.futures.pop(call_id, None)
        return Result.default_internal_error("RPC call timed out.")
    except Exception as e:
        self.futures.pop(call_id, None)
        return Result.default_internal_error(f"RPC call failed: {e}")

close() async

Closes the connection.

Source code in toolboxv2/mods/P2PRPCClient.py

async def close(self):
    """Closes the connection."""
    if self.writer:
        self.writer.close()
        await self.writer.wait_closed()
        print("RPC Client: Connection closed.")

connect() async

Connects to the local tcm instance and performs key exchange.

Source code in toolboxv2/mods/P2PRPCClient.py

async def connect(self):
    """Connects to the local tcm instance and performs key exchange."""
    try:
        self.reader, self.writer = await asyncio.open_connection(self.host, self.port)
        print(f"RPC Client: Connected to tcm at {self.host}:{self.port}")

        # Receive encrypted session key from server
        len_data = await self.reader.readexactly(4)
        encrypted_session_key_len = int.from_bytes(len_data, 'big')
        encrypted_session_key = (await self.reader.readexactly(encrypted_session_key_len)).decode('utf-8')

        # Decrypt session key using auth_key_part
        self.session_key = self.code.decrypt_symmetric(encrypted_session_key, self.auth_key_part)

        # Send challenge back to server, encrypted with session key
        challenge = "CHALLENGE_ACK"
        encrypted_challenge = self.code.encrypt_symmetric(challenge, self.session_key)
        self.writer.write(len(encrypted_challenge).to_bytes(4, 'big'))
        self.writer.write(encrypted_challenge.encode('utf-8'))
        await self.writer.drain()

        # Start a background task to listen for responses
        asyncio.create_task(self.listen_for_responses())

    except ConnectionRefusedError:
        print(f"RPC Client: Connection to {self.host}:{self.port} refused. Is the tcm peer running?")
        raise
    except Exception as e:
        print(f"RPC Client: Error during connection/key exchange: {e}")
        raise

listen_for_responses() async

Listens for incoming responses, decrypts them, and resolves the corresponding future.

Source code in toolboxv2/mods/P2PRPCClient.py

async def listen_for_responses(self):
    """Listens for incoming responses, decrypts them, and resolves the corresponding future."""
    try:
        while True:
            len_data = await self.reader.readexactly(4)
            msg_len = int.from_bytes(len_data, 'big')
            encrypted_msg_data = (await self.reader.readexactly(msg_len)).decode('utf-8')

            decrypted_msg_data = self.code.decrypt_symmetric(encrypted_msg_data, self.session_key)
            response = json.loads(decrypted_msg_data)

            call_id = response.get('call_id')
            if call_id in self.futures:
                future = self.futures.pop(call_id)
                future.set_result(response)
    except asyncio.IncompleteReadError:
        print("RPC Client: Connection closed.")
    except Exception as e:
        print(f"RPC Client: Error listening for responses: {e}")
    finally:
        # Clean up any pending futures
        for future in self.futures.values():
            future.set_exception(ConnectionError("Connection lost"))
        self.futures.clear()

test_rpc_client(app, host='127.0.0.1', port=8000, tb_r_key=None) async

An example of how to use the P2P RPC Client.

Source code in toolboxv2/mods/P2PRPCClient.py

@export(mod_name=Name, name="test_rpc_client", test=False)
async def test_rpc_client(app: App, host: str = '127.0.0.1', port: int = 8000, tb_r_key: str = None):
    """An example of how to use the P2P RPC Client."""
    if tb_r_key is None:
        tb_r_key = os.getenv("TB_R_KEY")
        if tb_r_key is None:
            raise ValueError("TB_R_KEY environment variable is not set.")

    client = P2PRPCClient(app, host, port, tb_r_key)
    try:
        await client.connect()
        # Example: Call the 'list-users' function from the 'helper' module
        result = await client.call("helper", "list-users")
        result.print()
    finally:
        await client.close()

P2PRPCServer

P2PRPCServer

Source code in toolboxv2/mods/P2PRPCServer.py

class P2PRPCServer:
    def __init__(self, app: App, host: str, port: int, tb_r_key: str, function_access_config: dict = None):
        self.app = app
        self.host = host
        self.port = port
        self.server = None
        self.code = Code()

        if len(tb_r_key) < 24:
            raise ValueError("TB_R_KEY must be at least 24 characters long for security.")
        self.auth_key_part = tb_r_key[:24]
        self.identification_part_server = tb_r_key[24:]

        self.function_access_config = function_access_config if function_access_config is not None else {}

    async def handle_client(self, reader, writer):
        """Callback to handle a single client connection from a tcm instance."""
        addr = writer.get_extra_info('peername')
        print(f"RPC Server: New connection from {addr}")

        session_key = self.code.generate_symmetric_key()
        encrypted_session_key = self.code.encrypt_symmetric(session_key, self.auth_key_part)

        try:
            writer.write(len(encrypted_session_key).to_bytes(4, 'big'))
            writer.write(encrypted_session_key.encode('utf-8'))
            await writer.drain()

            len_data = await reader.readexactly(4)
            encrypted_challenge_len = int.from_bytes(len_data, 'big')
            encrypted_challenge = (await reader.readexactly(encrypted_challenge_len)).decode('utf-8')

            decrypted_challenge = self.code.decrypt_symmetric(encrypted_challenge, session_key)
            if decrypted_challenge != "CHALLENGE_ACK":
                raise ValueError("Invalid challenge received.")

            print(f"RPC Server: Authenticated client {addr}")

            while True:
                len_data = await reader.readexactly(4)
                msg_len = int.from_bytes(len_data, 'big')

                encrypted_msg_data = (await reader.readexactly(msg_len)).decode('utf-8')

                decrypted_msg_data = self.code.decrypt_symmetric(encrypted_msg_data, session_key)

                response = await self.process_rpc(decrypted_msg_data, session_key)

                encrypted_response = self.code.encrypt_symmetric(json.dumps(response), session_key)

                writer.write(len(encrypted_response).to_bytes(4, 'big'))
                writer.write(encrypted_response.encode('utf-8'))
                await writer.drain()

        except asyncio.IncompleteReadError:
            print(f"RPC Server: Connection from {addr} closed.")
        except Exception as e:
            print(f"RPC Server: Error with client {addr}: {e}")
        finally:
            writer.close()
            await writer.wait_closed()

    async def process_rpc(self, msg_data: str, session_key: str) -> dict:
        """Processes a single RPC request and returns a response dictionary."""
        try:
            call = json.loads(msg_data)
            if call.get('type') != 'request':
                raise ValueError("Invalid message type")
        except (json.JSONDecodeError, ValueError) as e:
            return self.format_error(call.get('call_id'), -32700, f"Parse error: {e}")

        call_id = call.get('call_id')
        module = call.get('module')
        function = call.get('function')
        args = call.get('args', [])
        kwargs = call.get('kwargs', {})
        client_identification = call.get('identification_part')

        if not self.is_function_allowed(module, function, client_identification):
            error_msg = f"Function '{module}.{function}' is not allowed for identification '{client_identification}'."
            print(f"RPC Server: {error_msg}")
            return self.format_error(call_id, -32601, "Method not found or not allowed")

        print(f"RPC Server: Executing '{module}.{function}' for '{client_identification}'")
        try:
            result: Result = await self.app.a_run_any(
                (module, function),
                args_=args,
                kwargs_=kwargs,
                get_results=True
            )

            if result.is_error():
                return self.format_error(call_id, result.info.get('exec_code', -32000), result.info.get('help_text'), result.get())
            else:
                return {
                    "type": "response",
                    "call_id": call_id,
                    "result": result.get(),
                    "error": None
                }
        except Exception as e:
            print(f"RPC Server: Exception during execution of '{module}.{function}': {e}")
            return self.format_error(call_id, -32603, "Internal error during execution", str(e))

    def is_function_allowed(self, module: str, function: str, client_identification: str) -> bool:
        """Checks if a function is allowed for a given client identification."""
        if module not in self.function_access_config:
            return False

        allowed_functions_for_module = self.function_access_config[module]

        if function not in allowed_functions_for_module:
            return False

        # If the function is whitelisted, and there's a specific identification part,
        # you might want to add more granular control here.
        # For now, if it's in the whitelist, it's allowed for any identified client.
        # You could extend function_access_config to be:
        # {"ModuleName": {"function1": ["id1", "id2"], "function2": ["id3"]}}
        # For simplicity, current implementation assumes if module.function is in whitelist,
        # it's generally allowed for any authenticated client.
        return True

    def format_error(self, call_id, code, message, details=None) -> dict:
        """Helper to create a JSON-RPC error response object."""
        return {
            "type": "response",
            "call_id": call_id,
            "result": None,
            "error": {
                "code": code,
                "message": message,
                "details": details
            }
        }

    async def start(self):
        """Starts the TCP server."""
        self.server = await asyncio.start_server(
            self.handle_client, self.host, self.port
        )
        addr = self.server.sockets[0].getsockname()
        print(f"P2P RPC Server listening on {addr}")
        async with self.server:
            await self.server.serve_forever()

    def stop(self):
        """Stops the TCP server."""
        if self.server:
            self.server.close()
            print("P2P RPC Server stopped.")

format_error(call_id, code, message, details=None)

Helper to create a JSON-RPC error response object.

Source code in toolboxv2/mods/P2PRPCServer.py

def format_error(self, call_id, code, message, details=None) -> dict:
    """Helper to create a JSON-RPC error response object."""
    return {
        "type": "response",
        "call_id": call_id,
        "result": None,
        "error": {
            "code": code,
            "message": message,
            "details": details
        }
    }

handle_client(reader, writer) async

Callback to handle a single client connection from a tcm instance.

Source code in toolboxv2/mods/P2PRPCServer.py

async def handle_client(self, reader, writer):
    """Callback to handle a single client connection from a tcm instance."""
    addr = writer.get_extra_info('peername')
    print(f"RPC Server: New connection from {addr}")

    session_key = self.code.generate_symmetric_key()
    encrypted_session_key = self.code.encrypt_symmetric(session_key, self.auth_key_part)

    try:
        writer.write(len(encrypted_session_key).to_bytes(4, 'big'))
        writer.write(encrypted_session_key.encode('utf-8'))
        await writer.drain()

        len_data = await reader.readexactly(4)
        encrypted_challenge_len = int.from_bytes(len_data, 'big')
        encrypted_challenge = (await reader.readexactly(encrypted_challenge_len)).decode('utf-8')

        decrypted_challenge = self.code.decrypt_symmetric(encrypted_challenge, session_key)
        if decrypted_challenge != "CHALLENGE_ACK":
            raise ValueError("Invalid challenge received.")

        print(f"RPC Server: Authenticated client {addr}")

        while True:
            len_data = await reader.readexactly(4)
            msg_len = int.from_bytes(len_data, 'big')

            encrypted_msg_data = (await reader.readexactly(msg_len)).decode('utf-8')

            decrypted_msg_data = self.code.decrypt_symmetric(encrypted_msg_data, session_key)

            response = await self.process_rpc(decrypted_msg_data, session_key)

            encrypted_response = self.code.encrypt_symmetric(json.dumps(response), session_key)

            writer.write(len(encrypted_response).to_bytes(4, 'big'))
            writer.write(encrypted_response.encode('utf-8'))
            await writer.drain()

    except asyncio.IncompleteReadError:
        print(f"RPC Server: Connection from {addr} closed.")
    except Exception as e:
        print(f"RPC Server: Error with client {addr}: {e}")
    finally:
        writer.close()
        await writer.wait_closed()

is_function_allowed(module, function, client_identification)

Checks if a function is allowed for a given client identification.

Source code in toolboxv2/mods/P2PRPCServer.py

def is_function_allowed(self, module: str, function: str, client_identification: str) -> bool:
    """Checks if a function is allowed for a given client identification."""
    if module not in self.function_access_config:
        return False

    allowed_functions_for_module = self.function_access_config[module]

    if function not in allowed_functions_for_module:
        return False

    # If the function is whitelisted, and there's a specific identification part,
    # you might want to add more granular control here.
    # For now, if it's in the whitelist, it's allowed for any identified client.
    # You could extend function_access_config to be:
    # {"ModuleName": {"function1": ["id1", "id2"], "function2": ["id3"]}}
    # For simplicity, current implementation assumes if module.function is in whitelist,
    # it's generally allowed for any authenticated client.
    return True

process_rpc(msg_data, session_key) async

Processes a single RPC request and returns a response dictionary.

Source code in toolboxv2/mods/P2PRPCServer.py

async def process_rpc(self, msg_data: str, session_key: str) -> dict:
    """Processes a single RPC request and returns a response dictionary."""
    try:
        call = json.loads(msg_data)
        if call.get('type') != 'request':
            raise ValueError("Invalid message type")
    except (json.JSONDecodeError, ValueError) as e:
        return self.format_error(call.get('call_id'), -32700, f"Parse error: {e}")

    call_id = call.get('call_id')
    module = call.get('module')
    function = call.get('function')
    args = call.get('args', [])
    kwargs = call.get('kwargs', {})
    client_identification = call.get('identification_part')

    if not self.is_function_allowed(module, function, client_identification):
        error_msg = f"Function '{module}.{function}' is not allowed for identification '{client_identification}'."
        print(f"RPC Server: {error_msg}")
        return self.format_error(call_id, -32601, "Method not found or not allowed")

    print(f"RPC Server: Executing '{module}.{function}' for '{client_identification}'")
    try:
        result: Result = await self.app.a_run_any(
            (module, function),
            args_=args,
            kwargs_=kwargs,
            get_results=True
        )

        if result.is_error():
            return self.format_error(call_id, result.info.get('exec_code', -32000), result.info.get('help_text'), result.get())
        else:
            return {
                "type": "response",
                "call_id": call_id,
                "result": result.get(),
                "error": None
            }
    except Exception as e:
        print(f"RPC Server: Exception during execution of '{module}.{function}': {e}")
        return self.format_error(call_id, -32603, "Internal error during execution", str(e))

start() async

Starts the TCP server.

Source code in toolboxv2/mods/P2PRPCServer.py

async def start(self):
    """Starts the TCP server."""
    self.server = await asyncio.start_server(
        self.handle_client, self.host, self.port
    )
    addr = self.server.sockets[0].getsockname()
    print(f"P2P RPC Server listening on {addr}")
    async with self.server:
        await self.server.serve_forever()

stop()

Stops the TCP server.

Source code in toolboxv2/mods/P2PRPCServer.py

def stop(self):
    """Stops the TCP server."""
    if self.server:
        self.server.close()
        print("P2P RPC Server stopped.")

start_rpc_server(app, host='127.0.0.1', port=8888, tb_r_key=None, function_access_config=None) async

Starts the P2P RPC server.

Source code in toolboxv2/mods/P2PRPCServer.py

@export(mod_name=Name, name="start_server", test=False)
async def start_rpc_server(app: App, host: str = '127.0.0.1', port: int = 8888, tb_r_key: str = None, function_access_config: dict = None):
    """Starts the P2P RPC server."""
    if tb_r_key is None:
        tb_r_key = os.getenv("TB_R_KEY")
        if tb_r_key is None:
            raise ValueError("TB_R_KEY environment variable is not set.")

    server = P2PRPCServer(app, host, port, tb_r_key, function_access_config)
    try:
        await server.start()
    except KeyboardInterrupt:
        server.stop()

POA

module

ActionManagerEnhanced

Source code in toolboxv2/mods/POA/module.py

class ActionManagerEnhanced:
    DB_ITEMS_PREFIX = "donext_items"
    DB_HISTORY_PREFIX = "donext_history"
    DB_CURRENT_ITEM_PREFIX = "donext_current_item"
    DB_UNDO_LOG_PREFIX = "donext_undo_log"
    DB_SETTINGS_PREFIX = "donext_settings"  # Added for user settings

    def __init__(self, app: App, user_id: str):
        self.app = app
        self.user_id = user_id
        self.db = app.get_mod("DB")
        self.isaa = app.get_mod("isaa")

        self.settings: UserSettings = UserSettings(user_id=user_id)  # Initialize with defaults
        self.items: list[ActionItem] = []
        self.history: list[HistoryEntry] = []
        self.current_item: ActionItem | None = None
        self.undo_log: list[UndoLogEntry] = []

        self._load_settings()  # Load settings first as they might affect item loading
        self._load_data()

    def _get_db_key(self, prefix: str) -> str:
        return f"{prefix}_{self.user_id}"

    def get_user_timezone(self) -> pytz.BaseTzInfo:
        try:
            return pytz.timezone(self.settings.timezone)
        except pytz.UnknownTimeZoneError:
            return pytz.utc

    def _load_settings(self):
        settings_key = self._get_db_key(self.DB_SETTINGS_PREFIX)
        try:
            settings_data = self.db.get(settings_key)
            if settings_data.is_data() and settings_data.get():
                loaded_settings = json.loads(settings_data.get()[0]) if isinstance(settings_data.get(),
                                                                                   list) else json.loads(
                    settings_data.get())
                self.settings = UserSettings.model_validate_json_safe(loaded_settings)
            else:  # Save default settings if not found
                self._save_settings()
        except Exception as e:
            self.app.logger.error(f"Error loading settings for user {self.user_id}: {e}. Using defaults.")
            self.settings = UserSettings(user_id=self.user_id)  # Fallback to defaults
            self._save_settings()  # Attempt to save defaults

    def _save_settings(self):
        try:
            self.db.set(self._get_db_key(self.DB_SETTINGS_PREFIX), json.dumps(self.settings.model_dump_json_safe()))
        except Exception as e:
            self.app.logger.error(f"Error saving settings for user {self.user_id}: {e}")

    def update_user_settings(self, settings_data: dict[str, Any]) -> UserSettings:
        # Ensure user_id is not changed by malicious input
        current_user_id = self.settings.user_id
        updated_settings = UserSettings.model_validate(
            {**self.settings.model_dump(), **settings_data, "user_id": current_user_id})
        self.settings = updated_settings
        self._save_settings()
        # Potentially re-process items if timezone change affects interpretations, though this is complex.
        # For now, new items will use the new timezone. Existing UTC times remain.
        self.app.logger.info(f"User {self.user_id} settings updated: Timezone {self.settings.timezone}")
        return self.settings

    def _load_data(self):
        items_key = self._get_db_key(self.DB_ITEMS_PREFIX)
        history_key = self._get_db_key(self.DB_HISTORY_PREFIX)
        current_item_key = self._get_db_key(self.DB_CURRENT_ITEM_PREFIX)
        undo_log_key = self._get_db_key(self.DB_UNDO_LOG_PREFIX)
        user_tz_str = self.settings.timezone  # For model_validate_json_safe context

        try:
            items_data = self.db.get(items_key)
            if items_data.is_data() and items_data.get():
                loaded_items_raw = json.loads(items_data.get()[0]) if isinstance(items_data.get(),
                                                                                 list) else json.loads(items_data.get())
                self.items = [ActionItem.model_validate_json_safe(item_dict, user_timezone_str=user_tz_str) for
                              item_dict in loaded_items_raw]

            history_data = self.db.get(history_key)
            if history_data.is_data() and history_data.get():
                loaded_history_raw = json.loads(history_data.get()[0]) if isinstance(history_data.get(),
                                                                                     list) else json.loads(
                    history_data.get())
                self.history = [HistoryEntry.model_validate_json_safe(entry_dict) for entry_dict in loaded_history_raw]

            current_item_data = self.db.get(current_item_key)
            if current_item_data.is_data() and current_item_data.get():
                current_item_dict = json.loads(current_item_data.get()[0]) if isinstance(current_item_data.get(),
                                                                                         list) else json.loads(
                    current_item_data.get())
                if current_item_dict:
                    self.current_item = ActionItem.model_validate_json_safe(current_item_dict,
                                                                            user_timezone_str=user_tz_str)

            undo_log_data = self.db.get(undo_log_key)
            if undo_log_data.is_data() and undo_log_data.get():
                loaded_undo_raw = json.loads(undo_log_data.get()[0]) if isinstance(undo_log_data.get(),
                                                                                   list) else json.loads(
                    undo_log_data.get())
                self.undo_log = [UndoLogEntry.model_validate_json_safe(entry_dict) for entry_dict in loaded_undo_raw]

        except Exception as e:
            self.app.logger.error(f"Error loading data for user {self.user_id}: {e}")
            self.items, self.history, self.current_item, self.undo_log = [], [], None, []
        self._recalculate_next_due_for_all()

    def _save_data(self):
        try:
            self.db.set(self._get_db_key(self.DB_ITEMS_PREFIX),
                        json.dumps([item.model_dump_json_safe() for item in self.items]))
            self.db.set(self._get_db_key(self.DB_HISTORY_PREFIX),
                        json.dumps([entry.model_dump_json_safe() for entry in self.history]))
            self.db.set(self._get_db_key(self.DB_CURRENT_ITEM_PREFIX),
                        json.dumps(self.current_item.model_dump_json_safe() if self.current_item else None))
            self.db.set(self._get_db_key(self.DB_UNDO_LOG_PREFIX),
                        json.dumps([entry.model_dump_json_safe() for entry in self.undo_log]))
        except Exception as e:
            self.app.logger.error(f"Error saving data for user {self.user_id}: {e}")

    def _add_history_entry(self, item: ActionItem, status_override: ActionStatus | None = None,
                           notes: str | None = None):
        entry = HistoryEntry(
            item_id=item.id, item_title=item.title, item_type=item.item_type,
            status_changed_to=status_override or item.status,
            parent_id=item.parent_id, notes=notes
        )
        self.history.append(entry)

    def _datetime_to_user_tz(self, dt_utc: datetime | None) -> datetime | None:
        if not dt_utc: return None
        if dt_utc.tzinfo is None: dt_utc = pytz.utc.localize(dt_utc)  # Should already be UTC
        return dt_utc.astimezone(self.get_user_timezone())

    def _datetime_from_user_input_str(self, dt_str: str | None) -> datetime | None:
        if not dt_str: return None
        try:
            dt = isoparse(dt_str)
            if dt.tzinfo is None or dt.tzinfo.utcoffset(dt) is None:  # Naive
                return self.get_user_timezone().localize(dt).astimezone(pytz.utc)
            return dt.astimezone(pytz.utc)  # Aware, convert to UTC
        except ValueError:
            self.app.logger.warning(f"Could not parse datetime string: {dt_str}")
            return None

    def _recalculate_next_due(self, item: ActionItem):
        now_utc = datetime.now(pytz.utc)
        user_tz = self.get_user_timezone()

        if item.status == ActionStatus.COMPLETED and item.item_type == ItemType.TASK:
            if item.frequency and item.frequency != Frequency.ONE_TIME:
                base_time_utc = item.last_completed or now_utc  # last_completed is already UTC

                # If item had a fixed_time, align next_due to that time of day in user's timezone
                if item.fixed_time:
                    original_fixed_time_user_tz = item.fixed_time.astimezone(user_tz)
                    # Start from last_completed (or now if missing) in user's timezone for calculation
                    base_time_user_tz = base_time_utc.astimezone(user_tz)

                    # Ensure base_time_user_tz is at least original_fixed_time_user_tz for alignment
                    # but calculations should project from last completion.
                    # For example, if daily task due 9am was completed at 11am, next one is tomorrow 9am.
                    # If completed at 8am, next one is today 9am (if fixed_time was today 9am) or tomorrow 9am.

                    # Let's use last_completed as the primary anchor for when the *next* cycle starts.
                    # The original fixed_time's time component is used for the *time of day* of the next due.

                    current_anchor_user_tz = base_time_user_tz

                    # Calculate next occurrence based on frequency
                    if item.frequency == Frequency.DAILY:
                        next_due_user_tz_date = (current_anchor_user_tz + timedelta(days=1)).date()
                    elif item.frequency == Frequency.WEEKLY:
                        next_due_user_tz_date = (current_anchor_user_tz + timedelta(weeks=1)).date()
                    elif item.frequency == Frequency.MONTHLY:  # Simplified
                        next_due_user_tz_date = (current_anchor_user_tz + timedelta(days=30)).date()
                    elif item.frequency == Frequency.ANNUALLY:
                        next_due_user_tz_date = (current_anchor_user_tz + timedelta(days=365)).date()
                    else:  # Should not happen for recurring
                        item.next_due = None
                        return

                    # Combine with original time of day
                    next_due_user_tz = datetime.combine(next_due_user_tz_date, original_fixed_time_user_tz.time(),
                                                        tzinfo=user_tz)
                    item.next_due = next_due_user_tz.astimezone(pytz.utc)

                else:  # No original fixed_time, so recur based on current time of completion
                    if item.frequency == Frequency.DAILY:
                        item.next_due = base_time_utc + timedelta(days=1)
                    elif item.frequency == Frequency.WEEKLY:
                        item.next_due = base_time_utc + timedelta(weeks=1)
                    elif item.frequency == Frequency.MONTHLY:
                        item.next_due = base_time_utc + timedelta(days=30)
                    elif item.frequency == Frequency.ANNUALLY:
                        item.next_due = base_time_utc + timedelta(days=365)

                # Advance until future if needed (e.g., completing an overdue recurring task)
                # This loop must operate on user's local time perception of "next day"
                while item.next_due and item.next_due < now_utc:
                    next_due_user = item.next_due.astimezone(user_tz)
                    original_time_comp = next_due_user.time()  # Preserve time of day

                    if item.frequency == Frequency.DAILY:
                        next_due_user_adv = next_due_user + timedelta(days=1)
                    elif item.frequency == Frequency.WEEKLY:
                        next_due_user_adv = next_due_user + timedelta(weeks=1)
                    # For monthly/annually, simple timedelta might shift day of month. Using replace for date part.
                    elif item.frequency == Frequency.MONTHLY:
                        # This simplified logic might need dateutil.relativedelta for accuracy
                        year, month = (next_due_user.year, next_due_user.month + 1) if next_due_user.month < 12 else (
                            next_due_user.year + 1, 1)
                        try:
                            next_due_user_adv = next_due_user.replace(year=year, month=month)
                        except ValueError:  # Handle e.g. trying to set Feb 30
                            import calendar
                            last_day = calendar.monthrange(year, month)[1]
                            next_due_user_adv = next_due_user.replace(year=year, month=month, day=last_day)

                    elif item.frequency == Frequency.ANNUALLY:
                        try:
                            next_due_user_adv = next_due_user.replace(year=next_due_user.year + 1)
                        except ValueError:  # Handle leap day if original was Feb 29
                            next_due_user_adv = next_due_user.replace(year=next_due_user.year + 1,
                                                                      day=28)  # Or March 1st
                    else:
                        break

                    item.next_due = user_tz.localize(
                        datetime.combine(next_due_user_adv.date(), original_time_comp)).astimezone(pytz.utc)

                item.status = ActionStatus.NOT_STARTED  # Reset for next occurrence
            else:  # One-time task
                item.next_due = None
        elif item.status == ActionStatus.NOT_STARTED and item.fixed_time and not item.next_due:
            item.next_due = item.fixed_time  # fixed_time is already UTC

        # If task is not completed, not started, and has a next_due in the past, but also a fixed_time in the future
        # (e.g. recurring task whose current instance was missed, but fixed_time points to a specific time for all instances)
        # ensure next_due is not before fixed_time if fixed_time is relevant for setting.
        # This logic is complex. Current setup: fixed_time is the "template", next_due is the "instance".

    def _recalculate_next_due_for_all(self):
        for item in self.items:
            self._recalculate_next_due(item)

    def add_item(self, item_data: dict[str, Any], by_ai: bool = False, imported: bool = False) -> ActionItem:
        item_data['_user_timezone_str'] = self.settings.timezone  # For validation context
        item = ActionItem.model_validate(
            item_data)  # Pydantic handles string->datetime, then model_validator converts to UTC
        item.created_by_ai = by_ai
        item.updated_at = datetime.now(pytz.utc)  # Ensure update

        # Initial next_due for new items if not already set by iCal import logic
        if not item.next_due and item.fixed_time and item.status == ActionStatus.NOT_STARTED:
            item.next_due = item.fixed_time

        self.items.append(item)
        self._add_history_entry(item, status_override=ActionStatus.NOT_STARTED,
                                notes="Item created" + (" by AI" if by_ai else "") + (
                                    " via import" if imported else ""))
        if by_ai:
            self._log_ai_action("ai_create_item", [item.id])

        self._save_data()
        return item

    def get_item_by_id(self, item_id: str) -> ActionItem | None:
        return next((item for item in self.items if item.id == item_id), None)

    def update_item(self, item_id: str, update_data: dict[str, Any], by_ai: bool = False) -> ActionItem | None:
        item = self.get_item_by_id(item_id)
        if not item: return None

        previous_data_json = item.model_dump_json() if by_ai else None

        # Pass user timezone for validation context if datetime strings are present
        update_data_with_tz_context = {**update_data, '_user_timezone_str': self.settings.timezone}

        updated_item_dict = item.model_dump()
        updated_item_dict.update(update_data_with_tz_context)

        try:
            # Re-validate the whole model to ensure consistency and proper conversions
            new_item_state = ActionItem.model_validate(updated_item_dict)
            # Preserve original ID and created_at, apply new state
            new_item_state.id = item.id
            new_item_state.created_at = item.created_at
            self.items[self.items.index(item)] = new_item_state
            item = new_item_state
        except Exception as e:
            self.app.logger.error(f"Error validating updated item data: {e}. Update aborted for item {item_id}.")
            return None  # Or raise error

        item.updated_at = datetime.now(pytz.utc)
        item.created_by_ai = by_ai

        self._recalculate_next_due(item)
        self._add_history_entry(item, notes="Item updated" + (" by AI" if by_ai else ""))

        if by_ai:
            self._log_ai_action("ai_modify_item", [item.id],
                                {item.id: previous_data_json} if previous_data_json else None)

        self._save_data()
        return item

    def remove_item(self, item_id: str, record_history: bool = True) -> bool:
        item = self.get_item_by_id(item_id)
        if not item: return False

        children_ids = [child.id for child in self.items if child.parent_id == item_id]
        for child_id in children_ids:
            self.remove_item(child_id, record_history=record_history)

        self.items = [i for i in self.items if i.id != item_id]
        if self.current_item and self.current_item.id == item_id:
            self.current_item = None

        if record_history:
            self._add_history_entry(item, status_override=ActionStatus.CANCELLED, notes="Item removed")
        self._save_data()
        return True

    def set_current_item(self, item_id: str) -> ActionItem | None:
        item = self.get_item_by_id(item_id)
        if not item: return None
        if item.status == ActionStatus.COMPLETED and item.item_type == ItemType.TASK and item.frequency == Frequency.ONE_TIME:
            return None

        self.current_item = item
        if item.status == ActionStatus.NOT_STARTED:
            item.status = ActionStatus.IN_PROGRESS
            item.updated_at = datetime.now(pytz.utc)
            self._add_history_entry(item, notes="Set as current, status to In Progress")
        else:
            self._add_history_entry(item, notes="Set as current")
        self._save_data()
        return item

    def complete_current_item(self) -> ActionItem | None:
        if not self.current_item: return None

        item_to_complete = self.current_item
        item_to_complete.status = ActionStatus.COMPLETED
        item_to_complete.last_completed = datetime.now(pytz.utc)
        item_to_complete.updated_at = datetime.now(pytz.utc)

        self._recalculate_next_due(item_to_complete)
        self._add_history_entry(item_to_complete, status_override=ActionStatus.COMPLETED, notes="Marked as completed")

        self.current_item = None  # Clear current item after completion
        self._save_data()
        return item_to_complete

    def get_suggestions(self, count: int = 2) -> list[ActionItem]:
        # Prioritize AI suggestions if ISAA is available
        if self.isaa:
            active_items_for_ai = []
            for item in self.items:
                if item.status != ActionStatus.COMPLETED and item.status != ActionStatus.CANCELLED:
                    # Convert datetimes to user's local timezone string for AI context
                    item_dump = item.model_dump_json_safe()  # This is already UTC ISO
                    # Optionally, convert to user's timezone string if AI is better with local times
                    # For now, UTC ISO is fine.
                    active_items_for_ai.append(item_dump)

            MAX_ITEMS_FOR_CONTEXT = 20
            if len(active_items_for_ai) > MAX_ITEMS_FOR_CONTEXT:
                active_items_for_ai.sort(
                    key=lambda x: (x.get('priority', 3), x.get('next_due') or '9999-12-31T23:59:59Z'))
                active_items_for_ai = active_items_for_ai[:MAX_ITEMS_FOR_CONTEXT]

            now_user_tz_str = datetime.now(self.get_user_timezone()).isoformat()

            prompt = (
                f"User's current time: {now_user_tz_str} (Timezone: {self.settings.timezone}). "
                f"Active items (tasks/notes) are provided below (datetimes are in UTC ISO format). "
                f"Suggest the top {count} item IDs to focus on. Consider priority, due dates (next_due), "
                f"and if a current item is set (current_item_id), its sub-items might be relevant. "
                f"Tasks are generally more actionable. Focus on 'not_started' or 'in_progress'.\n\n"
                f"Active Items (JSON):\n{json.dumps(active_items_for_ai, indent=2)}\n\n"
                f"Current Item ID: {self.current_item.id if self.current_item else 'None'}\n\n"
                f"Return JSON: {{ \"suggested_item_ids\": [\"id1\", \"id2\"] }}."
            )

            class SuggestedIds(BaseModel):
                suggested_item_ids: list[str]

            try:
                structured_response = asyncio.run(
                    self.isaa.format_class(SuggestedIds, prompt, agent_name="TaskCompletion"))
                if structured_response and isinstance(structured_response, dict):
                    suggested_ids_model = SuggestedIds(**structured_response)
                    ai_suggestions = [self.get_item_by_id(id_str) for id_str in suggested_ids_model.suggested_item_ids
                                      if self.get_item_by_id(id_str)]
                    if ai_suggestions: return ai_suggestions[:count]
            except Exception as e:
                self.app.logger.error(f"Error getting AI suggestions: {e}")

        # Fallback to basic suggestions
        return self._get_basic_suggestions(count)

    def _get_basic_suggestions(self, count: int = 2) -> list[ActionItem]:
        now_utc = datetime.now(pytz.utc)
        available_items = [
            item for item in self.items
            if item.status in [ActionStatus.NOT_STARTED, ActionStatus.IN_PROGRESS]
        ]

        if self.current_item:
            sub_items = [item for item in available_items if item.parent_id == self.current_item.id]
            # If current item has actionable sub-items, prioritize them
            if any(s.next_due and s.next_due < (now_utc + timedelta(hours=2)) for s in sub_items) or \
                any(s.priority <= 2 for s in sub_items):  # Urgent sub-items (due soon or high priority)
                available_items = sub_items  # Focus on sub-items
            # If no urgent sub-items, consider other items too, but maybe give slight preference to other sub-items.
            # For simplicity now, if current_item is set, and it has sub-items, suggestions come from sub-items.
            # If no sub-items, or current_item is not set, consider all available_items.
            elif sub_items:  # Has sub-items, but none are "urgent" by above criteria
                available_items = sub_items
            # If current_item has no sub_items, then general pool is used.

        def sort_key(item: ActionItem):
            # Sort by: 1. Due Date (earlier is better, None is last) 2. Priority (lower num is higher)
            due_date_utc = item.next_due if item.next_due else datetime.max.replace(tzinfo=pytz.utc)
            return (due_date_utc, item.priority)

        available_items.sort(key=sort_key)
        return available_items[:count]

    def get_history(self, limit: int = 50) -> list[HistoryEntry]:
        return sorted(self.history, key=lambda x: x.timestamp, reverse=True)[:limit]

    def get_all_items_hierarchy(self) -> dict[str, list[dict[str, Any]]]:
        # This method remains largely the same, just ensure model_dump_json_safe is used.
        # Datetimes will be ISO UTC strings. Client JS needs to handle display in user's local time.
        hierarchy = {"root": []}
        item_map = {item.id: item.model_dump_json_safe() for item in self.items}  # Uses UTC ISO dates

        # This part seems fine, it builds hierarchy based on parent_id
        processed_ids = set()
        root_items_temp = []

        for _item_id, item_dict in item_map.items():
            parent_id = item_dict.get("parent_id")
            if parent_id and parent_id in item_map:
                if "children" not in item_map[parent_id]:
                    item_map[parent_id]["children"] = []
                item_map[parent_id]["children"].append(item_dict)
            else:
                root_items_temp.append(item_dict)
        hierarchy["root"] = root_items_temp

        def sort_children_recursive(node_list):
            for node_dict in node_list:
                if "children" in node_dict:
                    # Sort children by priority, then creation date
                    node_dict["children"].sort(key=lambda x: (x.get('priority', 3), isoparse(x.get('created_at'))))
                    sort_children_recursive(node_dict["children"])

        # Sort root items
        hierarchy["root"].sort(key=lambda x: (x.get('priority', 3), isoparse(x.get('created_at'))))
        sort_children_recursive(hierarchy["root"])
        return hierarchy

    # --- AI Specific Methods ---
    async def ai_create_item_from_text(self, text: str) -> ActionItem | None:
        if not self.isaa:
            self.app.logger.warning("ISAA module not available for AI item creation.")
            return None

        class ParsedItemFromText(BaseModel):
            item_type: Literal["task", "note"] = "task"
            title: str
            description: str | None = None
            priority: int | None = Field(default=3, ge=1, le=5)
            due_date_str: str | None = None  # e.g., "tomorrow", "next monday at 5pm", "2024-12-25 17:00"
            frequency_str: str | None = Field(default="one_time",
                                                 description="e.g. 'daily', 'weekly', 'one_time', 'every friday'")

        user_tz = self.get_user_timezone()
        current_time_user_tz_str = datetime.now(user_tz).strftime('%Y-%m-%d %H:%M:%S %Z%z')
        prompt = (
            f"User's current time is {current_time_user_tz_str}. Parse the input into a structured item. "
            f"For due_date_str, interpret relative dates/times based on this current time and output "
            f"a specific date string like 'YYYY-MM-DD HH:MM:SS'. If time is omitted, assume a default like 9 AM. "
            f"If date is omitted but time is given (e.g. 'at 5pm'), assume today if 5pm is future, else tomorrow. "
            f"User input: \"{text}\"\n\n"
            f"Format as JSON for ParsedItemFromText."
        )
        try:
            raw_response = await self.isaa.mini_task_completion(prompt, agent_name="TaskCompletion")
            if not raw_response: self.app.logger.error("AI parsing returned empty."); return None

            json_str = raw_response
            if "```json" in json_str: json_str = json_str.split("```json")[1].split("```")[0].strip()
            parsed_dict = json.loads(json_str)
            parsed_data_model = ParsedItemFromText(**parsed_dict)

            item_constructor_data = {
                "item_type": ItemType(parsed_data_model.item_type),
                "title": parsed_data_model.title,
                "description": parsed_data_model.description,
                "priority": parsed_data_model.priority or 3,
            }

            if parsed_data_model.due_date_str:
                # ISAA is prompted to return YYYY-MM-DD HH:MM:SS.
                # This string is assumed to be in the user's local timezone.
                # The ActionItem model_validator will convert this to UTC.
                item_constructor_data["fixed_time"] = parsed_data_model.due_date_str  # Pass as string

            # Frequency parsing (simplified)
            if parsed_data_model.frequency_str:
                freq_str_lower = parsed_data_model.frequency_str.lower()
                if "daily" in freq_str_lower:
                    item_constructor_data["frequency"] = Frequency.DAILY
                elif "weekly" in freq_str_lower:
                    item_constructor_data["frequency"] = Frequency.WEEKLY
                elif "monthly" in freq_str_lower:
                    item_constructor_data["frequency"] = Frequency.MONTHLY
                elif "annually" in freq_str_lower or "yearly" in freq_str_lower:
                    item_constructor_data["frequency"] = Frequency.ANNUALLY
                else:
                    item_constructor_data["frequency"] = Frequency.ONE_TIME

            return self.add_item(item_constructor_data, by_ai=True)
        except Exception as e:
            self.app.logger.error(
                f"Error creating item with AI: {e}. Raw: {raw_response if 'raw_response' in locals() else 'N/A'}")
            return None

    def _log_ai_action(self, action_type: Literal["ai_create_item", "ai_modify_item", "ical_import"],
                       item_ids: list[str], previous_data_map: dict[str, str] | None = None):
        entry = UndoLogEntry(action_type=action_type, item_ids=item_ids, previous_data_json_map=previous_data_map)
        self.undo_log.append(entry)
        if len(self.undo_log) > 20: self.undo_log = self.undo_log[-20:]
        # _save_data called by caller

    async def undo_last_ai_action(self) -> bool:  # Also handles iCal import undo
        if not self.undo_log: return False
        last_action = self.undo_log.pop()
        action_undone_count = 0

        if last_action.action_type in ["ai_create_item", "ical_import"]:
            for item_id in last_action.item_ids:
                if self.remove_item(item_id, record_history=False):  # Don't double-log removal for undo
                    action_undone_count += 1
        elif last_action.action_type == "ai_modify_item":
            if last_action.previous_data_json_map:
                for item_id, prev_data_json in last_action.previous_data_json_map.items():
                    try:
                        prev_data = ActionItem.model_validate_json_safe(json.loads(prev_data_json),
                                                                        user_timezone_str=self.settings.timezone)
                        # Replace item
                        found = False
                        for i, item_in_list in enumerate(self.items):
                            if item_in_list.id == item_id:
                                self.items[i] = prev_data
                                if self.current_item and self.current_item.id == item_id:
                                    self.current_item = prev_data
                                found = True
                                break
                        if found:
                            action_undone_count += 1
                        else:
                            self.app.logger.warning(f"Could not find item {item_id} to restore during AI undo.")
                    except Exception as e:
                        self.app.logger.error(f"Error restoring item {item_id} during undo: {e}")
            else:  # Should not happen for modify
                self.app.logger.warning(
                    f"Undo for AI modify action on item(s) {last_action.item_ids} had no previous_data_json_map.")

        if action_undone_count > 0:
            # Create a generic history entry for the undo action
            generic_undo_item_title = f"Related to {len(last_action.item_ids)} item(s)"
            if len(last_action.item_ids) == 1:
                item_for_title = self.get_item_by_id(last_action.item_ids[0])  # Might be None if it was a create undo
                generic_undo_item_title = item_for_title.title if item_for_title else "N/A (Undone Action)"

            self.history.append(HistoryEntry(
                item_id=last_action.item_ids[0],  # Representative item
                item_title=generic_undo_item_title,
                item_type=ItemType.TASK,  # Generic
                status_changed_to=ActionStatus.CANCELLED,  # Generic status for undo
                notes=f"Undid action: {last_action.action_type} for {len(last_action.item_ids)} item(s)."
            ))
            self._save_data()
            return True

        # If nothing was undone, put action back to log
        self.undo_log.append(last_action)
        return False

    # --- iCalendar Methods ---
    def _parse_ical_dt(self, dt_ical: vDatetime | vDate, user_tz: pytz.BaseTzInfo) -> datetime | None:
        """Converts icalendar vDatetime or vDate to UTC datetime."""
        if not dt_ical: return None
        dt_val = dt_ical.dt

        if isinstance(dt_val, datetime):
            if dt_val.tzinfo is None:  # Naive datetime, assume user's local timezone as per iCal spec for floating
                return user_tz.localize(dt_val).astimezone(pytz.utc)
            return dt_val.astimezone(pytz.utc)  # Aware datetime
        elif isinstance(dt_val, date):  # All-day event, represent as start of day in user's TZ, then UTC
            return user_tz.localize(datetime.combine(dt_val, datetime.min.time())).astimezone(pytz.utc)
        return None

    def _map_ical_priority_to_app(self, ical_priority: int | None) -> int:
        if ical_priority is None: return 3  # Default
        if 1 <= ical_priority <= 4: return 1  # High
        if ical_priority == 5: return 3  # Medium
        if 6 <= ical_priority <= 9: return 5  # Low
        return 3  # Default for 0 or other values

    def _map_app_priority_to_ical(self, app_priority: int) -> int:
        if app_priority == 1: return 1  # High
        if app_priority == 2: return 3
        if app_priority == 3: return 5  # Medium
        if app_priority == 4: return 7
        if app_priority == 5: return 9  # Low
        return 0  # No priority

    def _map_rrule_to_frequency(self, rrule_prop: vRecur | None) -> tuple[Frequency, str | None]:
        if not rrule_prop:
            return Frequency.ONE_TIME, None

        rrule_dict = rrule_prop.to_dict()
        freq = rrule_dict.get('FREQ')
        original_rrule_str = vRecur.from_dict(rrule_dict).to_ical().decode('utf-8')

        if freq == 'DAILY': return Frequency.DAILY, original_rrule_str
        if freq == 'WEEKLY': return Frequency.WEEKLY, original_rrule_str
        if freq == 'MONTHLY': return Frequency.MONTHLY, original_rrule_str
        if freq == 'YEARLY': return Frequency.ANNUALLY, original_rrule_str

        # If RRULE is complex or not a direct match, import as ONE_TIME for each instance
        # but store the original RRULE string for reference or future advanced handling.
        return Frequency.ONE_TIME, original_rrule_str

    def import_ical_events(self, ical_string: str) -> list[ActionItem]:
        imported_items: list[ActionItem] = []
        try:
            cal = iCalCalendar.from_ical(ical_string)
            user_tz = self.get_user_timezone()
            now_utc = datetime.now(pytz.utc)
            import_limit_date_utc = now_utc + timedelta(days=RECURRING_IMPORT_WINDOW_DAYS)

            processed_uids_for_session = set()  # To avoid processing same base recurring event multiple times in one import

            for component in cal.walk():
                if component.name == "VEVENT":
                    uid = component.get('uid')
                    if not uid:
                        uid = str(uuid.uuid4())  # Generate a UID if missing
                    else:
                        uid = uid.to_ical().decode('utf-8')

                    summary = component.get('summary', 'Untitled Event').to_ical().decode('utf-8')
                    description = component.get('description', '').to_ical().decode('utf-8')
                    location = component.get('location', '').to_ical().decode('utf-8')
                    dtstart_ical = component.get('dtstart')
                    dtend_ical = component.get('dtend')  # Can be used for duration if needed
                    ical_priority_val = component.get('priority')
                    ical_priority = int(ical_priority_val.to_ical().decode('utf-8')) if ical_priority_val else None

                    rrule_prop = component.get('rrule')  # This is a vRecur object or None

                    start_time_utc = self._parse_ical_dt(dtstart_ical, user_tz)
                    if not start_time_utc:
                        self.app.logger.warning(f"Skipping event '{summary}' due to missing/invalid DTSTART.")
                        continue

                    app_priority = self._map_ical_priority_to_app(ical_priority)

                    # Check for existing item with this iCal UID to potentially update (simplistic check)
                    # A more robust update would involve comparing sequence numbers, etc.
                    # For now, if UID exists, we might skip or update. Let's try to update.
                    # To keep it simpler for now, we will create new items for occurrences.
                    # UID management needs to be precise for updates.
                    # If an item is an instance of a recurring event, its UID in our system might be base_uid + occurrence_date.

                    if rrule_prop:
                        if uid in processed_uids_for_session:  # Already processed this recurring event's base
                            continue
                        processed_uids_for_session.add(uid)

                        # Handle recurring event
                        rrule_str = rrule_prop.to_ical().decode('utf-8')
                        # Ensure DTSTART is part of the rrule context if not explicitly in rrulestr
                        if 'DTSTART' not in rrule_str.upper() and start_time_utc:
                            # dateutil.rrule needs start time; icalendar often bakes it in.
                            # If start_time_utc is naive, use user_tz to make it aware.
                            dtstart_for_rrule = start_time_utc.astimezone(
                                user_tz) if start_time_utc.tzinfo else user_tz.localize(start_time_utc)
                            # rrule_obj = rrulestr(rrule_str, dtstart=dtstart_for_rrule) # This is complex due to TZ handling in rrulestr
                            # The icalendar library's component should be timezone aware from DTSTART
                            # So, let's assume dtstart_ical.dt is the correct starting point.
                            try:
                                rrule_obj = rrulestr(rrule_str, dtstart=dtstart_ical.dt)
                            except Exception as e_rr:
                                self.app.logger.error(
                                    f"Could not parse RRULE '{rrule_str}' for event '{summary}': {e_rr}")
                                continue

                        occurrences_imported = 0
                        # Generate occurrences starting from now (in user's timezone, aligned to event's time)
                        # or from event's start_time_utc if it's in the future.

                        # The rrule iteration should be in the event's original timezone context if possible,
                        # or consistently in user's timezone for 'now'.
                        # Let's use UTC for iteration and then convert.

                        # Iterate from the event's actual start time or now, whichever is later for relevant future instances.
                        iteration_start_utc = max(now_utc, start_time_utc)

                        for occ_dt_aware in rrule_obj.between(iteration_start_utc, import_limit_date_utc, inc=True):
                            if occurrences_imported >= MAX_RECURRING_INSTANCES_TO_IMPORT:
                                break

                            # occ_dt_aware is usually from dateutil.rrule, may need tzinfo set or conversion.
                            # If rrulestr was given an aware dtstart, occurrences should be aware.
                            # Ensure it's UTC for our system.
                            occ_utc = occ_dt_aware.astimezone(pytz.utc) if occ_dt_aware.tzinfo else pytz.utc.localize(
                                occ_dt_aware)

                            instance_uid = f"{uid}-{occ_utc.strftime('%Y%m%dT%H%M%S%Z')}"

                            # Check if this specific instance already exists
                            existing_instance = next((item for item in self.items if item.ical_uid == instance_uid),
                                                     None)
                            if existing_instance:
                                self.app.logger.info(
                                    f"Instance {instance_uid} for '{summary}' already exists. Skipping.")
                                continue

                            item_data = {
                                "title": summary, "description": description, "location": location,
                                "item_type": ItemType.TASK, "fixed_time": occ_utc,
                                "frequency": Frequency.ONE_TIME,  # Each imported instance is one-time in our system
                                "priority": app_priority, "ical_uid": instance_uid,  # Instance-specific UID
                                "status": ActionStatus.NOT_STARTED,
                                "ical_rrule_original": rrule_str  # Store original rule for reference
                            }
                            new_item = self.add_item(item_data, imported=True)
                            imported_items.append(new_item)
                            occurrences_imported += 1

                        if occurrences_imported == 0 and start_time_utc > now_utc and start_time_utc <= import_limit_date_utc:
                            # If it's a future non-recurring event (or rrule didn't yield instances in window but start is in window)
                            # This case is for when rrule_prop exists but yields no instances in the .between() range,
                            # but the initial DTSTART itself is valid and upcoming.
                            # However, rrule.between should include dtstart if inc=True and it's within range.
                            # This path might be redundant if .between is inclusive and dtstart is in range.
                            pass


                    else:  # Non-recurring event
                        # Only import if it's upcoming or started recently and not completed (e.g. within last day)
                        if start_time_utc < (
                            now_utc - timedelta(days=1)) and not dtend_ical:  # Too old, and no end time to check
                            self.app.logger.info(f"Skipping old non-recurring event '{summary}' (UID: {uid})")
                            continue
                        if dtend_ical:
                            end_time_utc = self._parse_ical_dt(dtend_ical, user_tz)
                            if end_time_utc and end_time_utc < now_utc:  # Event has already ended
                                self.app.logger.info(f"Skipping past event '{summary}' (UID: {uid}) that has ended.")
                                continue

                        existing_item = next((item for item in self.items if item.ical_uid == uid), None)
                        if existing_item:  # Simplistic update: remove old, add new. Better: update in place.
                            self.app.logger.info(
                                f"Event with UID {uid} ('{summary}') already exists. Re-importing (simple replace).")
                            self.remove_item(existing_item.id, record_history=False)

                        item_data = {
                            "title": summary, "description": description, "location": location,
                            "item_type": ItemType.TASK, "fixed_time": start_time_utc,
                            "frequency": Frequency.ONE_TIME, "priority": app_priority,
                            "ical_uid": uid, "status": ActionStatus.NOT_STARTED
                        }
                        new_item = self.add_item(item_data, imported=True)
                        imported_items.append(new_item)

            if imported_items:
                self._log_ai_action("ical_import", [item.id for item in imported_items])
            self._save_data()  # Ensure all changes are saved
            self.app.logger.info(f"Imported {len(imported_items)} items from iCalendar data.")

        except Exception as e:
            self.app.logger.error(f"Failed to parse iCalendar string: {e}", exc_info=True)
            # Potentially re-raise or return empty list with error status
        return imported_items

    def import_ical_from_url(self, url: str) -> list[ActionItem]:
        try:
            headers = {'User-Agent': 'POA_App/1.0 (+https://yourdomain.com/poa_app_info)'}  # Be a good internet citizen
            response = requests.get(url, timeout=10, headers=headers)
            response.raise_for_status()  # Raises HTTPError for bad responses (4XX or 5XX)
            return self.import_ical_events(response.text)
        except requests.exceptions.RequestException as e:
            self.app.logger.error(f"Error fetching iCalendar from URL {url}: {e}")
            return []
        except Exception as e:  # Catch other errors like parsing
            self.app.logger.error(f"Error processing iCalendar from URL {url}: {e}")
            return []

    def import_ical_from_file_content(self, file_content: bytes) -> list[ActionItem]:
        try:
            # Try to decode as UTF-8, but iCal can have other encodings.
            # Standard is UTF-8. `icalendar` lib handles encoding detection mostly.
            ical_string = file_content.decode('utf-8', errors='replace')
            return self.import_ical_events(ical_string)
        except UnicodeDecodeError as e:
            self.app.logger.error(f"Encoding error reading iCalendar file: {e}. Try ensuring UTF-8 encoding.")
            # Try with 'latin-1' as a common fallback for some older files
            try:
                ical_string = file_content.decode('latin-1', errors='replace')
                return self.import_ical_events(ical_string)
            except Exception as e_fallback:
                self.app.logger.error(f"Fallback decoding also failed for iCalendar file: {e_fallback}")
                return []
        except Exception as e:
            self.app.logger.error(f"Error processing iCalendar file content: {e}")
            return []

    def export_to_ical_string(self) -> str:
        cal = iCalCalendar()
        cal.add('prodid', '-//POA App//yourdomain.com//')
        cal.add('version', '2.0')
        user_tz = self.get_user_timezone()

        for item in self.items:
            if item.item_type == ItemType.TASK and item.fixed_time:
                event = iCalEvent()
                event.add('summary', item.title)

                # Ensure fixed_time is UTC for iCal standard practice
                dtstart_utc = item.fixed_time
                if dtstart_utc.tzinfo is None:  # Should not happen if stored correctly
                    dtstart_utc = pytz.utc.localize(dtstart_utc)
                else:
                    dtstart_utc = dtstart_utc.astimezone(pytz.utc)
                event.add('dtstart', dtstart_utc)  # vDatetime handles UTC conversion for .to_ical()

                # Add DTEND (e.g., 1 hour duration for tasks, or based on item if available)
                # For simplicity, let's assume 1 hour duration if not specified
                event.add('dtend', dtstart_utc + timedelta(hours=1))

                event.add('dtstamp', datetime.now(pytz.utc))  # Time the event was created in iCal
                event.add('uid', item.ical_uid or item.id)  # Use original iCal UID if present, else our ID

                if item.description:
                    event.add('description', item.description)
                if item.location:
                    event.add('location', item.location)

                event.add('priority', self._map_app_priority_to_ical(item.priority))

                # Handle recurrence
                if item.frequency != Frequency.ONE_TIME:
                    if item.ical_rrule_original:  # If we have the original complex rule, use it
                        try:
                            # vRecur.from_ical requires bytes
                            event.add('rrule', vRecur.from_ical(item.ical_rrule_original.encode()))
                        except Exception as e_rrule:
                            self.app.logger.warning(
                                f"Could not parse stored original RRULE '{item.ical_rrule_original}' for item {item.id}: {e_rrule}. Exporting as simple recurrence.")
                            # Fallback to simple mapping
                            self._add_simple_rrule(event, item.frequency)
                    else:  # Map simple frequency
                        self._add_simple_rrule(event, item.frequency)

                cal.add_component(event)
        return cal.to_ical().decode('utf-8')

    def _add_simple_rrule(self, event: iCalEvent, frequency: Frequency):
        rrule_params = {}
        if frequency == Frequency.DAILY:
            rrule_params['freq'] = 'DAILY'
        elif frequency == Frequency.WEEKLY:
            rrule_params['freq'] = 'WEEKLY'
        elif frequency == Frequency.MONTHLY:
            rrule_params['freq'] = 'MONTHLY'
        elif frequency == Frequency.ANNUALLY:
            rrule_params['freq'] = 'YEARLY'

        if rrule_params:
            event.add('rrule', vRecur(rrule_params))

SchedulerManager

SchedulerManagerClass

Source code in toolboxv2/mods/SchedulerManager.py

class SchedulerManagerClass:
    def __init__(self):
        self.jobs = {}
        self.thread = None
        self.running = False
        self.last_successful_jobs = deque(maxlen=3)  # Stores last 3 successful job names
        self.job_errors = {}  # Stores job names as keys and error messages as values

    def _run(self):
        while self.running:
            schedule.run_pending()
            time.sleep(1)

    def start(self):
        if not self.running:
            self.running = True
            self.thread = threading.Thread(target=self._run, daemon=True)
            self.thread.start()

    def stop(self):
        self.running = False
        if self.thread is not None:
            self.thread.join()

    def job_wrapper(self, job_name: str, job_function: callable):
        """
        Wrap a job function to track success and errors.
        """
        def wrapped_job(*args, **kwargs):
            try:
                job_function(*args, **kwargs)
                # If the job ran successfully, store it in the success queue
                self.last_successful_jobs.append(job_name)
                if job_name in self.job_errors:
                    del self.job_errors[job_name]  # Remove error record if job succeeded after failing
            except Exception as e:
                # Capture any exceptions and store them
                self.job_errors[job_name] = str(e)

        return wrapped_job


    def register_job(self,
                     job_id: str,
                     second: int = -1,
                     func: (Callable or str) | None = None,
                     job: schedule.Job | None = None,
                     time_passer: schedule.Job | None = None,
                     object_name: str | None = None,
                     receive_job: bool = False,
                     save: bool = False,
                     max_live: bool = False,
                     serializer=serializer_default,
                     args=None, kwargs=None):
        """
            Parameters
            ----------
                job_id : str
                    id for the job for management
                second : int
                    The time interval in seconds between each call of the job.
                func : Callable or str
                    The function to be executed as the job.
                job : schedule.Job
                    An existing job object from the schedule library.
                time_passer : schedule.Job
                    A job without a function, used to specify the time interval.
                object_name : str
                    The name of the object containing in the 'func' var to be executed.
                receive_job : bool
                    A flag indicating whether the job should be received from an object from 'func' var.
                save : bool
                    A flag indicating whether the job should be saved.
                max_live : bool
                    A flag indicating whether the job should have a maximum live time.
                serializer : dill
                    json pickel or dill must have a dumps fuction
                *args, **kwargs : Any serializable and deserializable
                    Additional arguments to be passed to the job function.

            Returns
            -------
           """

        if job is None and func is None:
            return Result.default_internal_error("Both job and func are not specified."
                                                 " Please specify either job or func.")
        if job is not None and func is not None:
            return Result.default_internal_error("Both job and func are specified. Please specify either job or func.")

        if job is not None:
            def func(x):
                return x
            return self._save_job(job_id=job_id,
                                  job=job,
                                  save=save,
                                  func=func,
                                  args=args,
                                  kwargs=kwargs,
                                  serializer=serializer)

        parsed_attr = self._parse_function(func=func, object_name=object_name)

        if parsed_attr.is_error():
            parsed_attr.result.data_info = f"Error parsing function for job : {job_id}"
            return parsed_attr

        if receive_job:
            job = parsed_attr.get()
        else:
            func = parsed_attr.get()

        time_passer = self._prepare_time_passer(time_passer=time_passer,
                                                second=second)

        job_func = self._prepare_job_func(func=func,
                                          max_live=max_live,
                                          second=second,
                                          args=args,
                                          kwargs=kwargs,
                                          job_id=job_id)

        job = self._get_final_job(job=job,
                                  func=self.job_wrapper(job_id, job_func),
                                  time_passer=time_passer,
                                  job_func=job_func,
                                  args=args,
                                  kwargs=kwargs)
        if job.is_error():
            return job

        job = job.get()

        return self._save_job(job_id=job_id,
                              job=job,
                              save=save,
                              func=func,
                              args=args,
                              kwargs=kwargs,
                              serializer=serializer)

    @staticmethod
    def _parse_function(func: str or Callable, object_name):
        if isinstance(func, str) and func.endswith('.py'):
            with open(func) as file:
                func_code = file.read()
                exec(func_code)
                func = locals()[object_name]
        elif isinstance(func, str) and func.endswith('.dill') and safety_mode == 'open':
            try:
                with open(func, 'rb') as file:
                    func = dill.load(file)
            except FileNotFoundError:
                return Result.default_internal_error(f"Function file {func} not found or dill not installed")
        elif isinstance(func, str):
            local_vars = {'app': get_app(from_=Name + f".pasing.{object_name}")}
            try:
                exec(func.strip(), {}, local_vars)
            except Exception as e:
                return Result.default_internal_error(f"Function parsing failed withe {e}")
            func = local_vars[object_name]
        elif isinstance(func, Callable):
            pass
        else:
            return Result.default_internal_error("Could not parse object scheduler_manager.parse_function")
        return Result.ok(func)

    @staticmethod
    def _prepare_time_passer(time_passer, second):
        if time_passer is None and second > 0:
            return schedule.every(second).seconds
        elif time_passer is None and second <= 0:
            raise ValueError("second must be greater than 0")
        return time_passer

    def _prepare_job_func(self, func: Callable, max_live: bool, second: float, job_id: str, *args, **kwargs):
        if max_live:
            end_time = datetime.now() + timedelta(seconds=second)

            def job_func():
                if datetime.now() < end_time:
                    func(*args, **kwargs)
                else:
                    job = self.jobs.get(job_id, {}).get('job')
                    if job is not None:
                        schedule.cancel_job(job)
                    else:
                        print("Error Canceling job")

            return job_func
        return func

    @staticmethod
    def _get_final_job(job, func, time_passer, job_func, args, kwargs):
        if job is None and isinstance(func, Callable):
            job = time_passer.do(job_func, *args, **kwargs)
        elif job is not None:
            pass
        else:
            return Result.default_internal_error("No Final job found for register")
        return Result.ok(job)

    def _save_job(self, job_id, job, save, args=None, **kwargs):
        if job is not None:
            self.jobs[job_id] = {'id': job_id, 'job': job, 'save': save, 'func': job_id, 'args': args,
                                 'kwargs': kwargs}
            f = (f"Added Job {job_id} :{' - saved' if save else ''}"
                  f"{' - args ' + str(len(args)) if args else ''}"
                  f"{' - kwargs ' + str(len(kwargs.keys())) if kwargs else ''}")
            return Result.ok(f)
        else:
            return Result.default_internal_error(job_id)

    def cancel_job(self, job_id):
        if job_id not in self.jobs:
            print("Job not found")
            return
        schedule.cancel_job(self.jobs[job_id].get('job'))
        self.jobs[job_id]["cancelled"] = True
        self.jobs[job_id]["save"] = False
        print("Job cancelled")

    def del_job(self, job_id):
        if job_id not in self.jobs:
            print("Job not found")
            return
        if not self.jobs[job_id].get("cancelled", False):
            print("Job not cancelled canceling job")
            self.cancel_job(job_id)
        del self.jobs[job_id]
        print("Job deleted")

    def save_jobs(self, file_path, serializer=serializer_default):
        with open(file_path, 'wb') as file:
            save_jobs = [job for job in self.jobs.values() if job['save']]
            serializer.dump(save_jobs, file)

    def load_jobs(self, file_path, deserializer=deserializer_default):
        with open(file_path, 'rb') as file:
            jobs = deserializer.load(file)
            for job_info in jobs:
                del job_info['job']
                func = deserializer.loads(job_info['func'])
                self.register_job(job_info['id'], func=func, **job_info)

    def get_tasks_table(self):
        if not self.jobs:
            return "No tasks registered."

        # Calculate the maximum width for each column
        id_width = max(len("Task ID"), max(len(job_id) for job_id in self.jobs))
        next_run_width = len("Next Execution")
        interval_width = len("Interval")

        # Create the header
        header = f"| {'Task ID':<{id_width}} | {'Next Execution':<{next_run_width}} | {'Interval':<{interval_width}} |"
        separator = f"|{'-' * (id_width + 2)}|{'-' * (next_run_width + 2)}|{'-' * (interval_width + 2)}|"

        # Create the table rows
        rows = []
        for job_id, job_info in self.jobs.items():
            job = job_info['job']
            next_run = job.next_run.strftime("%Y-%m-%d %H:%M:%S") if job.next_run else "N/A"
            interval = self._get_interval_str(job)
            row = f"| {job_id:<{id_width}} | {next_run:<{next_run_width}} | {interval:<{interval_width}} |"
            rows.append(row)

        # Combine all parts of the table
        table = "\n".join([header, separator] + rows)
        return table

    def _get_interval_str(self, job):
        if job.interval == 0:
            return "Once"

        units = [
            (86400, "day"),
            (3600, "hour"),
            (60, "minute"),
            (1, "second")
        ]

        for seconds, unit in units:
            if job.interval % seconds == 0:
                count = job.interval // seconds
                return f"Every {count} {unit}{'s' if count > 1 else ''}"

        return f"Every {job.interval} seconds"

job_wrapper(job_name, job_function)

Wrap a job function to track success and errors.

Source code in toolboxv2/mods/SchedulerManager.py

def job_wrapper(self, job_name: str, job_function: callable):
    """
    Wrap a job function to track success and errors.
    """
    def wrapped_job(*args, **kwargs):
        try:
            job_function(*args, **kwargs)
            # If the job ran successfully, store it in the success queue
            self.last_successful_jobs.append(job_name)
            if job_name in self.job_errors:
                del self.job_errors[job_name]  # Remove error record if job succeeded after failing
        except Exception as e:
            # Capture any exceptions and store them
            self.job_errors[job_name] = str(e)

    return wrapped_job

register_job(job_id, second=-1, func=None, job=None, time_passer=None, object_name=None, receive_job=False, save=False, max_live=False, serializer=serializer_default, args=None, kwargs=None)

Parameters

job_id : str
    id for the job for management
second : int
    The time interval in seconds between each call of the job.
func : Callable or str
    The function to be executed as the job.
job : schedule.Job
    An existing job object from the schedule library.
time_passer : schedule.Job
    A job without a function, used to specify the time interval.
object_name : str
    The name of the object containing in the 'func' var to be executed.
receive_job : bool
    A flag indicating whether the job should be received from an object from 'func' var.
save : bool
    A flag indicating whether the job should be saved.
max_live : bool
    A flag indicating whether the job should have a maximum live time.
serializer : dill
    json pickel or dill must have a dumps fuction
*args, **kwargs : Any serializable and deserializable
    Additional arguments to be passed to the job function.

Returns

Source code in toolboxv2/mods/SchedulerManager.py

def register_job(self,
                 job_id: str,
                 second: int = -1,
                 func: (Callable or str) | None = None,
                 job: schedule.Job | None = None,
                 time_passer: schedule.Job | None = None,
                 object_name: str | None = None,
                 receive_job: bool = False,
                 save: bool = False,
                 max_live: bool = False,
                 serializer=serializer_default,
                 args=None, kwargs=None):
    """
        Parameters
        ----------
            job_id : str
                id for the job for management
            second : int
                The time interval in seconds between each call of the job.
            func : Callable or str
                The function to be executed as the job.
            job : schedule.Job
                An existing job object from the schedule library.
            time_passer : schedule.Job
                A job without a function, used to specify the time interval.
            object_name : str
                The name of the object containing in the 'func' var to be executed.
            receive_job : bool
                A flag indicating whether the job should be received from an object from 'func' var.
            save : bool
                A flag indicating whether the job should be saved.
            max_live : bool
                A flag indicating whether the job should have a maximum live time.
            serializer : dill
                json pickel or dill must have a dumps fuction
            *args, **kwargs : Any serializable and deserializable
                Additional arguments to be passed to the job function.

        Returns
        -------
       """

    if job is None and func is None:
        return Result.default_internal_error("Both job and func are not specified."
                                             " Please specify either job or func.")
    if job is not None and func is not None:
        return Result.default_internal_error("Both job and func are specified. Please specify either job or func.")

    if job is not None:
        def func(x):
            return x
        return self._save_job(job_id=job_id,
                              job=job,
                              save=save,
                              func=func,
                              args=args,
                              kwargs=kwargs,
                              serializer=serializer)

    parsed_attr = self._parse_function(func=func, object_name=object_name)

    if parsed_attr.is_error():
        parsed_attr.result.data_info = f"Error parsing function for job : {job_id}"
        return parsed_attr

    if receive_job:
        job = parsed_attr.get()
    else:
        func = parsed_attr.get()

    time_passer = self._prepare_time_passer(time_passer=time_passer,
                                            second=second)

    job_func = self._prepare_job_func(func=func,
                                      max_live=max_live,
                                      second=second,
                                      args=args,
                                      kwargs=kwargs,
                                      job_id=job_id)

    job = self._get_final_job(job=job,
                              func=self.job_wrapper(job_id, job_func),
                              time_passer=time_passer,
                              job_func=job_func,
                              args=args,
                              kwargs=kwargs)
    if job.is_error():
        return job

    job = job.get()

    return self._save_job(job_id=job_id,
                          job=job,
                          save=save,
                          func=func,
                          args=args,
                          kwargs=kwargs,
                          serializer=serializer)

Tools

Bases: MainTool, SchedulerManagerClass

Source code in toolboxv2/mods/SchedulerManager.py

class Tools(MainTool, SchedulerManagerClass):
    version = version

    def __init__(self, app=None):
        self.name = Name
        self.color = "VIOLET2"

        self.keys = {"mode": "db~mode~~:"}
        self.encoding = 'utf-8'
        self.tools = {'name': Name}

        SchedulerManagerClass.__init__(self)
        MainTool.__init__(self,
                          load=self.init_sm,
                          v=self.version,
                          name=self.name,
                          color=self.color,
                          on_exit=self.on_exit)


    @export(
        mod_name=Name,
        name="Version",
        version=version,
    )
    def get_version(self):
        return self.version

    # Exportieren der Scheduler-Instanz für die Nutzung in anderen Modulen
    @export(mod_name=Name, name='init', version=version, initial=True)
    def init_sm(self):
        if os.path.exists(self.app.data_dir + '/jobs.compact'):
            print("SchedulerManager try loading from file")
            self.load_jobs(
                self.app.data_dir + '/jobs.compact'
            )
            print("SchedulerManager Successfully loaded")
        print("STARTING SchedulerManager")
        self.start()

    @export(mod_name=Name, name='clos_manager', version=version, exit_f=True)
    def on_exit(self):
        self.stop()
        self.save_jobs(self.app.data_dir + '/jobs.compact')
        return f"saved {len(self.jobs.keys())} jobs in {self.app.data_dir + '/jobs.compact'}"

    @export(mod_name=Name, name='instance', version=version)
    def get_instance(self):
        return self

    @export(mod_name=Name, name='start', version=version)
    def start_instance(self):
        return self.start()

    @export(mod_name=Name, name='stop', version=version)
    def stop_instance(self):
        return self.stop()

    @export(mod_name=Name, name='cancel', version=version)
    def cancel_instance(self, job_id):
        return self.cancel_job(job_id)

    @export(mod_name=Name, name='dealt', version=version)
    def dealt_instance(self, job_id):
        return self.del_job(job_id)

    @export(mod_name=Name, name='add', version=version)
    def register_instance(self, job_data: dict):
        """
        example dicts :
            -----------
            {
                "job_id": "job0",
                "second": 0,
                "func": None,
                "job": None,
                "time_passer": None,
                "object_name": "tb_job_fuction",
                "receive_job": False,
                "save": False,
                "max_live": True,
                # just lev it out "serializer": serializer_default,
                "args": [],
                "kwargs": {},
            }

            job_id : str
                id for the job for management
            second (optional): int
                The time interval in seconds between each call of the job.
            func (optional): Callable or str
                The function to be executed as the job.
            job (optional):  schedule.Job
                An existing job object from the schedule library.
            time_passer (optional):  schedule.Job
                A job without a function, used to specify the time interval.
            object_name (optional): str
                The name of the object containing in the 'func' var to be executed.
            receive_job (optional): bool
                A flag indicating whether the job should be received from an object from 'func' var.
            save (optional): bool
                A flag indicating whether the job should be saved.
            max_live (optional): bool
                A flag indicating whether the job should have a maximum live time.
            serializer (optional): bool
                json pickel or dill must have a dumps fuction
            *args, **kwargs (optional):
                Additional arguments to be passed to the job function.


        Parameters
            ----------
           job_data : dict

        example usage
            ----------
            `python

            `

    """
        if job_data is None:
            self.app.logger.error("No job data provided")
            return None
        job_id = job_data["job_id"]
        second = job_data.get("second", 0)
        func = job_data.get("func")
        job = job_data.get("job")
        time_passer = job_data.get("time_passer")
        object_name = job_data.get("object_name", "tb_job_fuction")
        receive_job = job_data.get("receive_job", False)
        save = job_data.get("save", False)
        max_live = job_data.get("max_live", True)
        serializer = job_data.get("serializer", serializer_default)
        args = job_data.get("args", ())
        kwargs = job_data.get("kwargs", {})

        return self.register_job(
            job_id=job_id,
            second=second,
            func=func,
            job=job,
            time_passer=time_passer,
            object_name=object_name,
            receive_job=receive_job,
            save=save,
            max_live=max_live,
            serializer=serializer,
            args=args,
            kwargs=kwargs
        )

register_instance(job_data)

example dicts

---

{ "job_id": "job0", "second": 0, "func": None, "job": None, "time_passer": None, "object_name": "tb_job_fuction", "receive_job": False, "save": False, "max_live": True, # just lev it out "serializer": serializer_default, "args": [], "kwargs": {}, }

job_id : str id for the job for management second (optional): int The time interval in seconds between each call of the job. func (optional): Callable or str The function to be executed as the job. job (optional): schedule.Job An existing job object from the schedule library. time_passer (optional): schedule.Job A job without a function, used to specify the time interval. object_name (optional): str The name of the object containing in the 'func' var to be executed. receive_job (optional): bool A flag indicating whether the job should be received from an object from 'func' var. save (optional): bool A flag indicating whether the job should be saved. max_live (optional): bool A flag indicating whether the job should have a maximum live time. serializer (optional): bool json pickel or dill must have a dumps fuction args, *kwargs (optional): Additional arguments to be passed to the job function.

Parameters ---------- job_data : dict

example usage ---------- `python

`

Source code in toolboxv2/mods/SchedulerManager.py

@export(mod_name=Name, name='add', version=version)
def register_instance(self, job_data: dict):
    """
    example dicts :
        -----------
        {
            "job_id": "job0",
            "second": 0,
            "func": None,
            "job": None,
            "time_passer": None,
            "object_name": "tb_job_fuction",
            "receive_job": False,
            "save": False,
            "max_live": True,
            # just lev it out "serializer": serializer_default,
            "args": [],
            "kwargs": {},
        }

        job_id : str
            id for the job for management
        second (optional): int
            The time interval in seconds between each call of the job.
        func (optional): Callable or str
            The function to be executed as the job.
        job (optional):  schedule.Job
            An existing job object from the schedule library.
        time_passer (optional):  schedule.Job
            A job without a function, used to specify the time interval.
        object_name (optional): str
            The name of the object containing in the 'func' var to be executed.
        receive_job (optional): bool
            A flag indicating whether the job should be received from an object from 'func' var.
        save (optional): bool
            A flag indicating whether the job should be saved.
        max_live (optional): bool
            A flag indicating whether the job should have a maximum live time.
        serializer (optional): bool
            json pickel or dill must have a dumps fuction
        *args, **kwargs (optional):
            Additional arguments to be passed to the job function.


    Parameters
        ----------
       job_data : dict

    example usage
        ----------
        `python

        `

"""
    if job_data is None:
        self.app.logger.error("No job data provided")
        return None
    job_id = job_data["job_id"]
    second = job_data.get("second", 0)
    func = job_data.get("func")
    job = job_data.get("job")
    time_passer = job_data.get("time_passer")
    object_name = job_data.get("object_name", "tb_job_fuction")
    receive_job = job_data.get("receive_job", False)
    save = job_data.get("save", False)
    max_live = job_data.get("max_live", True)
    serializer = job_data.get("serializer", serializer_default)
    args = job_data.get("args", ())
    kwargs = job_data.get("kwargs", {})

    return self.register_job(
        job_id=job_id,
        second=second,
        func=func,
        job=job,
        time_passer=time_passer,
        object_name=object_name,
        receive_job=receive_job,
        save=save,
        max_live=max_live,
        serializer=serializer,
        args=args,
        kwargs=kwargs
    )

SocketManager

The SocketManager Supports 2 types of connections 1. Client Server 2. Peer to Peer

TruthSeeker

arXivCrawler

ArXiv Crawler for TruthSeeker. Main module for processing research queries.

ArXivPDFProcessor

Main processor for research queries. This is a wrapper around the new ResearchProcessor for backward compatibility.

Source code in toolboxv2/mods/TruthSeeker/arXivCrawler.py

class ArXivPDFProcessor:
    """
    Main processor for research queries.
    This is a wrapper around the new ResearchProcessor for backward compatibility.
    """
    def __init__(self,
                 query: str,
                 tools,
                 chunk_size: int = 1_000_000,
                 overlap: int = 2_000,
                 max_workers=None,
                 num_search_result_per_query=6,
                 max_search=6,
                 download_dir="pdfs",
                 callback=None,
                 num_workers=None):
        """Initialize the ArXiv PDF processor.

        Args:
            query: Research query
            tools: Tools module
            chunk_size: Size of text chunks for processing
            overlap: Overlap between chunks
            max_workers: Maximum number of worker threads
            num_search_result_per_query: Number of search results per query
            max_search: Maximum number of search queries
            download_dir: Directory to save downloaded files
            callback: Callback function for status updates
            num_workers: Number of worker threads
        """
        # Create the new research processor
        self.processor = ResearchProcessor(
            query=query,
            tools=tools,
            chunk_size=chunk_size,
            overlap=overlap,
            max_workers=max_workers,
            num_search_result_per_query=num_search_result_per_query,
            max_search=max_search,
            download_dir=download_dir,
            callback=callback,
            num_workers=num_workers
        )

        # Copy attributes for backward compatibility
        self.insights_generated = False
        self.queries_generated = False
        self.query = query
        self.tools = tools
        self.mem = tools.get_memory()
        self.chunk_size = chunk_size
        self.overlap = overlap
        self.max_workers = max_workers
        self.nsrpq = num_search_result_per_query
        self.max_search = max_search
        self.download_dir = download_dir
        self.parser = RobustPDFDownloader(download_dir=download_dir)
        self.callback = callback if callback is not None else lambda status: None
        self.mem_name = None
        self.current_session = None
        self.all_ref_papers = 0
        self.last_insights_list = None
        self.all_texts_len = 0
        self.f_texts_len = 0
        self.s_id = str(uuid.uuid4())
        self.semantic_model = self.processor.semantic_model
        self._query_progress = {}
        self._progress_lock = threading.Lock()
        self.num_workers = self.processor.num_workers

    def _update_global_progress(self) -> float:
        """Calculate overall progress considering all processing phases."""
        return self.processor._update_global_progress()

    async def search_and_process_papers(self, queries: list[str]) -> list[Paper]:
        """Search for and process papers based on queries.

        Args:
            queries: List of search queries

        Returns:
            List of processed papers
        """
        # Use the new processor to search and process papers
        unified_papers = await self.processor.search_and_process_papers(queries)

        # Convert UnifiedPaper objects to Paper objects for backward compatibility
        papers = []
        for paper in unified_papers:
            if paper.source == "arxiv":
                # Convert to the old Paper format
                arxiv_paper = Paper(
                    title=paper.title,
                    authors=paper.authors,
                    summary=paper.summary,
                    url=paper.url,
                    pdf_url=paper.pdf_url,
                    published=paper.published,
                    updated=paper.source_specific_data.get("updated", ""),
                    categories=paper.source_specific_data.get("categories", []),
                    paper_id=paper.paper_id
                )
                papers.append(arxiv_paper)

        # Update attributes for backward compatibility
        self.all_ref_papers = self.processor.all_ref_papers
        self.all_texts_len = self.processor.all_texts_len
        self.f_texts_len = self.processor.f_texts_len

        return papers

    def send_status(self, step: str, progress: float = None, additional_info: str = ""):
        """Send status update via callback."""
        if progress is None:
            progress = self._update_global_progress()
        self.callback({
            "step": step,
            "progress": progress,
            "info": additional_info
        })

    def generate_queries(self) -> list[str]:
        self.send_status("Generating search queries")
        self.queries_generated = False

        class ArXivQueries(BaseModel):
            queries: list[str] = Field(..., description="List of ArXiv search queries (en)")

        try:
            query_generator: ArXivQueries = self.tools.format_class(
                ArXivQueries,
                f"Generate a list of precise ArXiv search queries to comprehensively address: {self.query}"
            )
            queries = [self.query] + query_generator["queries"]
        except Exception:
            self.send_status("Error generating queries", additional_info="Using default query.")
            queries = [self.query]

        if len(queries[:self.max_search]) > 0:
            self.queries_generated = True
        return queries[:self.max_search]

    def init_process_papers(self):
        self.mem.create_memory(self.mem_name, model_config={"model_name": "anthropic/claude-3-5-haiku-20241022"})
        self.send_status("Memory initialized")


    async def generate_insights(self, queries) -> dict:
        self.send_status("Generating insights")
        query = self.query
        # max_it = 0
        results = await self.mem.query(query=query, memory_names=self.mem_name, unified_retrieve=True, query_params={
            "max_sentences": 25})
        #query = queries[min(len(queries)-1, max_it)]

        self.insights_generated = True
        self.send_status("Insights generated", progress=1.0)
        return results

    async def extra_query(self, query, query_params=None, unified_retrieve=True):
        self.send_status("Processing follow-up query", progress=0.5)
        results = await self.mem.query(query=query, memory_names=self.mem_name,
                                                      query_params=query_params, unified_retrieve=unified_retrieve)
        self.send_status("Processing follow-up query Done", progress=1)
        return results

    def generate_mem_name(self):
        class UniqueMemoryName(BaseModel):
            """unique memory name based on the user query"""
            name: str
        return self.tools.get_agent("thinkm").format_class(UniqueMemoryName, self.query).get('name', '_'.join(self.query.split(" ")[:3]))

    def initialize(self, session_id, second=False):
        self.current_session = session_id
        self.insights_generated = False
        self.queries_generated = False
        if second:
            return
        self.mem_name = self.generate_mem_name().strip().replace("\n", '') + '_' + session_id
        self.init_process_papers()

    async def process(self, query=None) -> tuple[list[Paper], dict]:
        if query is not None:
            self.query = query
        self.send_status("Starting research process")
        t0 = time.perf_counter()
        self.initialize(self.s_id, query is not None)

        queries = self.generate_queries()

        papers = await self.search_and_process_papers(queries)

        if len(papers) == 0:
            class UserQuery(BaseModel):
                """Fix all typos and clear the original user query"""
                new_query: str
            self.query= self.tools.format_class(
                UserQuery,
                self.query
            )["new_query"]
            queries = self.generate_queries()
            papers = await self.search_and_process_papers(queries)

        insights = await self.generate_insights(queries)

        elapsed_time = time.perf_counter() - t0
        self.send_status("Process complete", progress=1.0,
                         additional_info=f"Total time: {elapsed_time:.2f}s, Papers analyzed: {len(papers)}/{self.all_ref_papers}")

        return papers, insights

    @staticmethod
    def estimate_processing_metrics(query_length: int, **config) -> (float, float):
        """Return estimated time (seconds) and price for processing."""
        total_papers = config['max_search'] * config['num_search_result_per_query']
        median_text_length = 100000  # 10 pages * 10000 characters

        # Estimated chunks to process
        total_chunks = total_papers * (median_text_length / config['chunk_size']) + 1 / config['overlap']
        processed_chunks = total_chunks * 0.45
        total_chars = TextSplitter(config['chunk_size'],
                     config['overlap']
                     ).approximate(config['chunk_size'] * processed_chunks)
        # Time estimation (seconds)
        .75 / config['chunk_size']  # Hypothetical time per chunk in seconds
        w = (config.get('num_workers', 16) if config.get('num_workers', 16) is not None else 16 / 10)
        # Processing_ time - Insights Genration - Insights Query   -   Indexing Time     -    Download Time     -       workers   -   Query Genration time - Ui - Init Db
        estimated_time = ((8+total_papers*0.012)+(total_chunks/20000) * .005 + (total_chunks/2) * .0003 + total_papers * 2.8 ) / w + (0.25 * config['max_search']) + 6 + 4

        price_per_char = 0.0000012525
        price_per_t_chunk =  total_chars * price_per_char
        estimated_price = price_per_t_chunk ** 1.7

        # estimated_price = 0 if query_length < 420 and estimated_price < 5 else estimated_price
        if estimated_time < 10:
            estimated_time = 10
        if estimated_price < .04:
            estimated_price = .04
        return round(estimated_time, 2), round(estimated_price, 4)

__init__(query, tools, chunk_size=1000000, overlap=2000, max_workers=None, num_search_result_per_query=6, max_search=6, download_dir='pdfs', callback=None, num_workers=None)

Initialize the ArXiv PDF processor.

Parameters:

Name	Type	Description	Default
query	str	Research query	required
tools		Tools module	required
chunk_size	int	Size of text chunks for processing	1000000
overlap	int	Overlap between chunks	2000
max_workers		Maximum number of worker threads	None
num_search_result_per_query		Number of search results per query	6
max_search		Maximum number of search queries	6
download_dir		Directory to save downloaded files	'pdfs'
callback		Callback function for status updates	None
num_workers		Number of worker threads	None

Source code in toolboxv2/mods/TruthSeeker/arXivCrawler.py

def __init__(self,
             query: str,
             tools,
             chunk_size: int = 1_000_000,
             overlap: int = 2_000,
             max_workers=None,
             num_search_result_per_query=6,
             max_search=6,
             download_dir="pdfs",
             callback=None,
             num_workers=None):
    """Initialize the ArXiv PDF processor.

    Args:
        query: Research query
        tools: Tools module
        chunk_size: Size of text chunks for processing
        overlap: Overlap between chunks
        max_workers: Maximum number of worker threads
        num_search_result_per_query: Number of search results per query
        max_search: Maximum number of search queries
        download_dir: Directory to save downloaded files
        callback: Callback function for status updates
        num_workers: Number of worker threads
    """
    # Create the new research processor
    self.processor = ResearchProcessor(
        query=query,
        tools=tools,
        chunk_size=chunk_size,
        overlap=overlap,
        max_workers=max_workers,
        num_search_result_per_query=num_search_result_per_query,
        max_search=max_search,
        download_dir=download_dir,
        callback=callback,
        num_workers=num_workers
    )

    # Copy attributes for backward compatibility
    self.insights_generated = False
    self.queries_generated = False
    self.query = query
    self.tools = tools
    self.mem = tools.get_memory()
    self.chunk_size = chunk_size
    self.overlap = overlap
    self.max_workers = max_workers
    self.nsrpq = num_search_result_per_query
    self.max_search = max_search
    self.download_dir = download_dir
    self.parser = RobustPDFDownloader(download_dir=download_dir)
    self.callback = callback if callback is not None else lambda status: None
    self.mem_name = None
    self.current_session = None
    self.all_ref_papers = 0
    self.last_insights_list = None
    self.all_texts_len = 0
    self.f_texts_len = 0
    self.s_id = str(uuid.uuid4())
    self.semantic_model = self.processor.semantic_model
    self._query_progress = {}
    self._progress_lock = threading.Lock()
    self.num_workers = self.processor.num_workers

estimate_processing_metrics(query_length, **config) staticmethod

Return estimated time (seconds) and price for processing.

Source code in toolboxv2/mods/TruthSeeker/arXivCrawler.py

@staticmethod
def estimate_processing_metrics(query_length: int, **config) -> (float, float):
    """Return estimated time (seconds) and price for processing."""
    total_papers = config['max_search'] * config['num_search_result_per_query']
    median_text_length = 100000  # 10 pages * 10000 characters

    # Estimated chunks to process
    total_chunks = total_papers * (median_text_length / config['chunk_size']) + 1 / config['overlap']
    processed_chunks = total_chunks * 0.45
    total_chars = TextSplitter(config['chunk_size'],
                 config['overlap']
                 ).approximate(config['chunk_size'] * processed_chunks)
    # Time estimation (seconds)
    .75 / config['chunk_size']  # Hypothetical time per chunk in seconds
    w = (config.get('num_workers', 16) if config.get('num_workers', 16) is not None else 16 / 10)
    # Processing_ time - Insights Genration - Insights Query   -   Indexing Time     -    Download Time     -       workers   -   Query Genration time - Ui - Init Db
    estimated_time = ((8+total_papers*0.012)+(total_chunks/20000) * .005 + (total_chunks/2) * .0003 + total_papers * 2.8 ) / w + (0.25 * config['max_search']) + 6 + 4

    price_per_char = 0.0000012525
    price_per_t_chunk =  total_chars * price_per_char
    estimated_price = price_per_t_chunk ** 1.7

    # estimated_price = 0 if query_length < 420 and estimated_price < 5 else estimated_price
    if estimated_time < 10:
        estimated_time = 10
    if estimated_price < .04:
        estimated_price = .04
    return round(estimated_time, 2), round(estimated_price, 4)

search_and_process_papers(queries) async

Search for and process papers based on queries.

Parameters:

Name	Type	Description	Default
queries	list[str]	List of search queries	required

Returns:

Type	Description
list[Paper]	List of processed papers

Source code in toolboxv2/mods/TruthSeeker/arXivCrawler.py

async def search_and_process_papers(self, queries: list[str]) -> list[Paper]:
    """Search for and process papers based on queries.

    Args:
        queries: List of search queries

    Returns:
        List of processed papers
    """
    # Use the new processor to search and process papers
    unified_papers = await self.processor.search_and_process_papers(queries)

    # Convert UnifiedPaper objects to Paper objects for backward compatibility
    papers = []
    for paper in unified_papers:
        if paper.source == "arxiv":
            # Convert to the old Paper format
            arxiv_paper = Paper(
                title=paper.title,
                authors=paper.authors,
                summary=paper.summary,
                url=paper.url,
                pdf_url=paper.pdf_url,
                published=paper.published,
                updated=paper.source_specific_data.get("updated", ""),
                categories=paper.source_specific_data.get("categories", []),
                paper_id=paper.paper_id
            )
            papers.append(arxiv_paper)

    # Update attributes for backward compatibility
    self.all_ref_papers = self.processor.all_ref_papers
    self.all_texts_len = self.processor.all_texts_len
    self.f_texts_len = self.processor.f_texts_len

    return papers

send_status(step, progress=None, additional_info='')

Send status update via callback.

Source code in toolboxv2/mods/TruthSeeker/arXivCrawler.py

def send_status(self, step: str, progress: float = None, additional_info: str = ""):
    """Send status update via callback."""
    if progress is None:
        progress = self._update_global_progress()
    self.callback({
        "step": step,
        "progress": progress,
        "info": additional_info
    })

main(query='Beste strategien in bretspielen sitler von katar') async

Main execution function

Source code in toolboxv2/mods/TruthSeeker/arXivCrawler.py

async def main(query: str = "Beste strategien in bretspielen sitler von katar"):
    """Main execution function"""
    with Spinner("Init Isaa"):
        tools = get_app("ArXivPDFProcessor", name=None).get_mod("isaa")
        tools.init_isaa(build=True)
    processor = ArXivPDFProcessor(query, tools=tools)
    papers, insights = await processor.process()

    print("Generated Insights:", insights)
    print("Generated Insights_list:", processor.last_insights_list)
    kb = tools.get_memory(processor.mem_name)
    print(await kb.query_concepts("AI"))
    print(await kb.retrieve("Evaluation metrics for assessing AI Agent performance"))
    print(kb.concept_extractor.concept_graph.concepts.keys())
    kb.vis(output_file="insights_graph.html")
    kb.save("mem.plk")
    # await get_app("ArXivPDFProcessor", name=None).a_idle()
    return insights

nGui

import colorsys import json import time from datetime import datetime, timedelta from queue import Queue from typing import Dict, Union, List, Any

import os import random from threading import Thread, Event

import networkx as nx from dataclasses import asdict

from toolboxv2 import get_app from toolboxv2.mods.FastApi.fast_nice import register_nicegui

import asyncio

from nicegui import ui

from pathlib import Path import stripe

from toolboxv2.mods.TruthSeeker.arXivCrawler import Paper from toolboxv2.mods.isaa.base.AgentUtils import anything_from_str_to_dict

Set your secret key (use environment variables in production!)

stripe.api_key = os.getenv('STRIPE_SECRET_KEY', 'sk_test_YourSecretKey')

def create_landing_page(): # Set up dynamic background ui.query("body").style("background: linear-gradient(135deg, #1a1a2e 0%, #16213e 100%)")

# Main container with enhanced responsive design
with ui.column().classes(
"w-full max-w-md p-8 rounded-3xl shadow-2xl "
"items-center self-center mx-auto my-8"
):
    # Advanced styling for glass-morphism effect
    ui.query(".nicegui-column").style("""
    background: rgba(255, 255, 255, 0.05);
    backdrop-filter: blur(12px);
    border: 1px solid rgba(255, 255, 255, 0.1);
    transition: all 0.3s ease-in-out;
    """)

    # Animated logo/brand icon
    with ui.element("div").classes("animate-fadeIn"):
        ui.icon("science").classes(
        "text-7xl mb-6 text-primary "
        "transform hover:scale-110 transition-transform"
        )

    # Enhanced typography for title
    ui.label("TruthSeeker").classes(
    "text-5xl font-black text-center "
    "text-primary mb-2 animate-slideDown"
    )

    # Stylized subtitle with brand message
    ui.label("Precision. Discovery. Insights.").classes(
    "text-xl font-medium text-center "
    "mb-10 animate-fadeIn"
    )

    # Button container for consistent spacing
    ui.button(
    "Start Research",
    on_click=lambda: ui.navigate.to("/open-Seeker.seek")
    ).classes(
    "w-full px-6 py-4 text-lg font-bold "
    "bg-primary hover:bg-primary-dark "
    "transform hover:-translate-y-0.5 "
    "transition-all duration-300 ease-in-out "
    "rounded-xl shadow-lg animate-slideUp"
    )

    # Navigation links container
    with ui.element("div").classes("mt-8 space-y-3 text-center"):
        ui.link(
        "Demo video",
        ).classes(
        "block text-lg text-gray-200 hover:text-primary "
        "transition-colors duration-300 animate-fadeIn"
        ).on("click", lambda: ui.navigate.to("/open-Seeker.demo"))

        ui.link(
        "About Us",
        ).classes(
        "block text-lg text-gray-400 hover:text-primary "
        "transition-colors duration-300 animate-fadeIn"
        ).on("click", lambda: ui.navigate.to("/open-Seeker.about"))

def create_video_demo(): with ui.card().classes('w-full max-w-3xl mx-auto').style( 'background: var(--background-color); color: var(--text-color)'): # Video container with responsive aspect ratio with ui.element('div').classes('relative w-full aspect-video'): video = ui.video('../api/TruthSeeker/video').classes('w-full h-full object-cover')

        # Custom controls overlay
        with ui.element('div').classes('absolute bottom-0 left-0 right-0 bg-black/50 p-2'):
            with ui.row().classes('items-center gap-2'):
                #play_btn = ui.button(icon='play_arrow', on_click=lambda: video.props('playing=true'))
                #pause_btn = ui.button(icon='pause', on_click=lambda: video.props('playing=false'))
                ui.slider(min=0, max=100, value=0).classes('w-full').bind_value(video, 'time')
                #mute_btn = ui.button(icon='volume_up', on_click=lambda: video.props('muted=!muted'))
                #fullscreen_btn = ui.button(icon='fullscreen', on_click=lambda: video.props('fullscreen=true'))


    # Video description
    ui.markdown('Walkthrough of TruthSeeker features and capabilities.')
    # Back to Home Button
    ui.button('Back to Home', on_click=lambda: ui.navigate.to('/open-Seeker')).classes(
        'mt-6 w-full bg-primary text-white hover:opacity-90'
    )

return video

def create_about_page(): """Create a comprehensive About page for TruthSeeker""" with ui.column().classes('w-full max-w-4xl mx-auto p-6'): # Page Header ui.label('About TruthSeeker').classes('text-4xl font-bold text-primary mb-6')

    # Mission Statement
    with ui.card().classes('w-full mb-6').style(
        'background: var(--background-color); color: var(--text-color); padding: 20px; border-radius: 8px; box-shadow: 0 2px 8px rgba(0,0,0,0.1);'
    ):
        ui.label('Our Mission').classes('text-2xl font-semibold text-primary mb-4')
        ui.markdown("""
            TruthSeeker aims to democratize access to scientific knowledge,
            transforming complex academic research into comprehensible insights.
            We bridge the gap between raw data and meaningful understanding.
        """).classes('text-lg').style('color: var(--text-color);')

    # Core Technologies
    with ui.card().classes('w-full mb-6').style(
        'background: var(--background-color); color: var(--text-color); padding: 20px; border-radius: 8px; box-shadow: 0 2px 8px rgba(0,0,0,0.1);'
    ):
        ui.label('Core Technologies').classes('text-2xl font-semibold text-primary mb-4')
        with ui.row().classes('gap-4 w-full'):
            with ui.column().classes('flex-1 text-center'):
                ui.icon('search').classes('text-4xl text-primary mb-2')
                ui.label('Advanced Query Processing').classes('font-bold')
                ui.markdown('Intelligent algorithms that extract nuanced research insights.').style(
                    'color: var(--text-color);')
            with ui.column().classes('flex-1 text-center'):
                ui.icon('analytics').classes('text-4xl text-primary mb-2')
                ui.label('Semantic Analysis').classes('font-bold')
                ui.markdown('Deep learning models for comprehensive research verification.').style(
                    'color: var(--text-color);')
            with ui.column().classes('flex-1 text-center'):
                ui.icon('verified').classes('text-4xl text-primary mb-2')
                ui.label('Research Validation').classes('font-bold')
                ui.markdown('Multi-layered verification of academic sources.').style('color: var(--text-color);')
    # Research Process
    with ui.card().classes('w-full').style('background: var(--background-color);color: var(--text-color);'):
        ui.label('Research Discovery Process').classes('text-2xl font-semibold text-primary mb-4')
        with ui.card().classes('q-pa-md q-mx-auto').style(
            'max-width: 800px; background: var(--background-color); border-radius: 8px; box-shadow: 0 2px 8px rgba(0,0,0,0.1);'
        ) as card:
            ui.markdown("# Research Workflow").style(
                "color: var(--primary-color); text-align: center; margin-bottom: 20px;")
            ui.markdown(
                """
                Welcome to TruthSeeker’s interactive research assistant. Follow the steps below to transform your initial inquiry into a refined, actionable insight.
                """
            ).style("color: var(--text-color); text-align: center; margin-bottom: 30px;")

            # The stepper component
            with ui.stepper().style('background: var(--background-color); color: var(--text-color);') as stepper:
                # Step 1: Query Initialization
                with ui.step('Query Initialization'):
                    ui.markdown("### Step 1: Query Initialization").style("color: var(--primary-color);")
                    ui.markdown(
                        """
                        Begin by entering your research question or selecting from popular academic domains.
                        This sets the direction for our semantic analysis engine.
                        """
                    ).style("color: var(--text-color); margin-bottom: 20px;")
                    with ui.stepper_navigation():
                        ui.button('Next', on_click=stepper.next).props('rounded color=primary')

                # Step 2: Semantic Search
                with ui.step('Semantic Search'):
                    ui.markdown("### Step 2: Semantic Search").style("color: var(--primary-color);")
                    ui.markdown(
                        """
                        Our advanced algorithms now process your input to generate context-rich queries.
                        This stage refines the search context by understanding the deeper intent behind your question.
                        """
                    ).style("color: var(--text-color); margin-bottom: 20px;")
                    with ui.stepper_navigation():
                        ui.button('Back', on_click=stepper.previous).props('flat')
                        ui.button('Next', on_click=stepper.next).props('rounded color=primary')

                # Step 3: Document Analysis
                with ui.step('Document Analysis'):
                    ui.markdown("### Step 3: Document Analysis").style("color: var(--primary-color);")
                    ui.markdown(
                        """
                        The system then dives into a detailed analysis of academic papers, parsing content to extract key insights and connections.
                        This ensures that even subtle but crucial information is captured.
                        """
                    ).style("color: var(--text-color); margin-bottom: 20px;")
                    with ui.stepper_navigation():
                        ui.button('Back', on_click=stepper.previous).props('flat')
                        ui.button('Next', on_click=stepper.next).props('rounded color=primary')

                # Step 4: Insight Generation
                with ui.step('Insight Generation'):
                    ui.markdown("### Step 4: Insight Generation").style("color: var(--primary-color);")
                    ui.markdown(
                        """
                        Finally, we synthesize the analyzed data into clear, actionable research summaries.
                        These insights empower you with concise guidance to drive further inquiry or practical application.
                        """
                    ).style("color: var(--text-color); margin-bottom: 20px;")
                    with ui.stepper_navigation():
                        ui.button('Back', on_click=stepper.previous).props('flat')

    # Back to Home Button
    ui.button('Back to Home', on_click=lambda: ui.navigate.to('/open-Seeker')).classes(
        'mt-6 w-full bg-primary text-white hover:opacity-90'
    )

Dummy-Implementierung für get_tools()

def get_tools(): """ Hier solltest du dein richtiges Werkzeug-Objekt zurückliefern. In diesem Beispiel gehen wir davon aus, dass du über eine Funktion wie get_app verfügst. """ return get_app("ArXivPDFProcessor", name=None).get_mod("isaa")

def create_graph_tab(processor_instance: Dict, graph_ui: ui.element, main_ui: ui.element): """Create and update the graph visualization"""

# Get HTML graph from processor
_html_content = processor_instance["instance"].tools.get_memory(processor_instance["instance"].mem_name)
html_content = "" if isinstance(_html_content, list) else _html_content.vis(get_output_html=True)

# Ensure static directory exists
static_dir = Path('dist/static')
static_dir.mkdir(exist_ok=True)

# Save HTML to static file
graph_file = static_dir / f'graph{processor_instance["instance"].mem_name}.html'
# Save HTML to static file with added fullscreen functionality

# Add fullscreen JavaScript
graph_file.write_text(html_content, encoding='utf-8')

with main_ui:
    # Clear existing content except fullscreen button
    graph_ui.clear()

    with graph_ui:
        ui.html(f"""

            <iframe
                 src="/static/graph{processor_instance["instance"].mem_name}.html"
                style="width: 100%; height: 800px; border: none; background: #1a1a1a;"
                >
            </iframe>
        """).classes('w-full h-full')

is_init = [False]

--- Database Setup ---

def get_db(): db = get_app().get_mod("DB") if not is_init[0]: is_init[0] = True db.edit_cli("LD") db.initialize_database() return db

import pickle

--- Session State Management ---

def get_user_state(session_id: str, is_new=False) -> dict: db = get_db() state_ = { 'balance': .5, 'last_reset': datetime.utcnow().isoformat(), 'research_history': [], 'payment_id': '', } if session_id is None: state_['balance'] *= -1 if is_new: return state_, True return state_ state = db.get(f"TruthSeeker::session:{session_id}") if state.get() is None: state = state_ if is_new: return state_, True else: try: state = pickle.loads(state.get()) except Exception as e: print(e) state = { 'balance': 0.04, 'last_reset': datetime.utcnow().isoformat(), 'research_history': ["Sorry we had an error recreating your state"], 'payment_id': '', } if is_new: return state, True if is_new: return state, False return state

def save_user_state(session_id: str, state: dict): db = get_db() print("Saving state") db.set(f"TruthSeeker::session:{session_id}", pickle.dumps(state)).print()

def delete_user_state(session_id: str): db = get_db() print("Saving state") db.delete(f"TruthSeeker::session:{session_id}").print()

def reset_daily_balance(state: dict, valid=False) -> dict: now = datetime.utcnow() last_reset = datetime.fromisoformat(state.get('last_reset', now.isoformat())) if now - last_reset > timedelta(hours=24): state['balance'] = max(state.get('balance', 1.6 if valid else 0.5), 1.6 if valid else 0.5) state['last_reset'] = now.isoformat() return state

class MemoryResultsDisplay

def init(self, results: List[Dict[str, Any]], main_ui: ui.element): self.results = results self.main_ui = main_ui self.setup_ui()

def setup_ui(self): """Set up the main UI for displaying memory results""" with self.main_ui: self.main_ui.clear() with ui.column().classes('w-full'): for mem_result in self.results: self.create_memory_card(mem_result)

def create_memory_card(self, mem_result: Dict[str, Any]): """Create a card for each memory result""" result = mem_result.get("result", {}) with self.main_ui: if isinstance(result, dict): self.display_dict_result(result) elif hasattr(result, 'overview'): # Assuming RetrievalResult type self.display_retrieval_result(result) else: ui.label("Unsupported result type").classes('--text-color:error')

def display_dict_result(self, result: Dict[str, Any]): """Display dictionary-based result with collapsible sections""" # Summary Section summary = result.get("summary", {}) if isinstance(summary, str): try: summary = json.loads(summary[:-1]) except json.JSONDecodeError: summary = {"error": "Could not parse summary"}

# Raw Results Section
raw_results = result.get("raw_results", {})
if isinstance(raw_results, str):
    try:
        raw_results = json.loads(raw_results[:-1])
    except json.JSONDecodeError:
        raw_results = {"error": "Could not parse raw results"}

# Metadata Section
metadata = result.get("metadata", {})
with self.main_ui:
    # Collapsible Sections
    with ui.column().classes('w-full space-y-2').style("max-width: 100%;"):
        # Summary Section
        with ui.expansion('Summary', icon='description').classes('w-full') as se:
            self.display_nested_data(summary, main_ui=se)

        # Raw Results Section
        with ui.expansion('Raw Results', icon='work').classes('w-full') as re:
            self.display_nested_data(raw_results, main_ui=re)

        # Metadata Section
        if metadata:
            with ui.expansion('Metadata', icon='info').classes('w-full'):
                ui.markdown(f"```json

{json.dumps(metadata, indent=2)} ```").style("max-width: 100%;")

def display_retrieval_result(self, result):
    """Display retrieval result with detailed sections"""
    with self.main_ui:
        with ui.column().classes('w-full space-y-4').style("max-width: 100%;"):
            # Overview Section
            with ui.expansion('Overview', icon='visibility').classes('w-full') as ov:
                for overview_item in result.overview:
                    if isinstance(overview_item, str):
                        overview_item = json.loads(overview_item)
                    self.display_nested_data(overview_item, main_ui=ov)

            # Details Section
            with ui.expansion('Details', icon='article').classes('w-full'):
                for chunk in result.details:
                    with ui.card().classes('w-full p-3 mb-2').style("background: var(--background-color)"):
                        ui.label(chunk.text).classes('font-medium mb-2 --text-color:secondary')

                        with ui.row().classes('w-full justify-between').style("background: var(--background-color)"):
                            ui.label(f"Embedding Shape: {chunk.embedding.shape}").classes('text-sm')
                            ui.label(f"Content Hash: {chunk.content_hash}").classes('text-sm')

                        if chunk.cluster_id is not None:
                            ui.label(f"Cluster ID: {chunk.cluster_id}").classes('text-sm')

            # Cross References Section
            with ui.expansion('Cross References', icon='link').classes('w-full'):
                for topic, chunks in result.cross_references.items():
                    with ui.card().classes('w-full p-3 mb-2').style("background: var(--background-color)"):
                        ui.label(topic).classes('font-semibold mb-2 --text-color:secondary')
                        for chunk in chunks:
                            ui.label(chunk.text).classes('text-sm mb-1')

def display_nested_data(self, data: Union[Dict, List], indent: int = 0, main_ui=None):
    """Recursively display nested dictionary or list data"""
    with (self.main_ui if main_ui is None else main_ui):
        if isinstance(data, dict):
            with ui.column().classes(f'ml-{indent * 2}').style("max-width: 100%;"):
                for key, value in data.items():
                    with ui.row().classes('items-center'):
                        ui.label(f"{key}:").classes('font-bold mr-2 --text-color:primary')
                        if isinstance(value, list):
                            if key == "main_chunks":
                                continue
                            self.display_nested_data(value, indent + 1, main_ui=main_ui)
                        if isinstance(value, dict):
                            ui.markdown(f"```json

{json.dumps(value, indent=2)} ").classes("break-words w-full").style("max-width: 100%;") else: ui.label(str(value)).classes('--text-color:secondary') elif isinstance(data, list): with ui.column().classes(f'ml-{indent * 2}').style("max-width: 100%;"): for item in data: if isinstance(item, str): item = json.loads(item) if isinstance(item, list): self.display_nested_data(item, indent + 1, main_ui=main_ui) if isinstance(item, dict): ui.markdown(f"json {json.dumps(item, indent=2)} ```").classes("break-words w-full").style("max-width: 100%;") else: ui.label(str(item)).classes('--text-color:secondary')

def create_followup_section(processor_instance: Dict, main_ui: ui.element, session_id, balance): main_ui.clear() with main_ui: ui.label("Query Interface (1ct)").classes("text-xl font-semibold mb-4")

    # Container for query inputs
    query_container = ui.column().classes("w-full gap-4")
    query = ""  # Store references to query inputs
    # Query parameters section
    with ui.expansion("Query Parameters", icon="settings").classes("w-full") as query_e:
        with ui.grid(columns=2).classes("w-full gap-4"):
            k_input = ui.number("Results Count (k)", value=2, min=1, max=20)
            min_sim = ui.number("Min Similarity", value=.3, min=0, max=1, step=0.1)
            cross_depth = ui.number("Cross Reference Depth", value=2, min=1, max=5)
            max_cross = ui.number("Max Cross References", value=10, min=1, max=20)
            max_sent = ui.number("Max Sentences", value=10, min=1, max=50)
            unified = ui.switch("Unified Retrieve (+3ct)", value=True)

    # Results display
    with ui.element("div").classes("w-full mt-4") as results_display:
        pass
    results_display = results_display
    with query_container:
        query_input = ui.input("Query", placeholder="Enter your query...")                 .classes("w-full")
    # Control buttons
    with ui.row().classes("w-full gap-4 mt-4"):
        ui.button("Execute Query", on_click=lambda: asyncio.create_task(execute_query()))                 .classes("bg-green-600 hover:bg-green-700")
        ui.button("Clear Results", on_click=lambda: results_display.clear())                 .classes("bg-red-600 hover:bg-red-700")
query_input = query_input

async def execute_query():
    """Execute a single query with parameters"""
    nonlocal query_input, results_display, main_ui
    try:
        query_text = query_input.value
        if not query_text.strip():
            with main_ui:
                ui.notify("No Input", type="warning")
            return ""

        if not processor_instance.get("instance"):
            with main_ui:
                ui.notify("No active processor instance", type="warning")
            return
        # Collect parameters
        params = {
            "k": int(k_input.value),
            "min_similarity": min_sim.value,
            "cross_ref_depth": int(cross_depth.value),
            "max_cross_refs": int(max_cross.value),
            "max_sentences": int(max_sent.value),
            "unified": unified.value
        }
        # Construct query parameters
        query_params = {
            "k": params["k"],
            "min_similarity": params["min_similarity"],
            "cross_ref_depth": params["cross_ref_depth"],
            "max_cross_refs": params["max_cross_refs"],
            "max_sentences": params["max_sentences"]
        }

        # Execute query
        results = await processor_instance["instance"].extra_query(
            query=query_text,
            query_params=query_params,
            unified_retrieve=params["unified"]
        )
        print("results",results)
        s = get_user_state(session_id)
        s['balance'] -= .04 if unified.value else .01
        save_user_state(session_id, s)
        with main_ui:
            balance.set_text(f"Balance: {s['balance']:.2f}€")
        # Format results
        with main_ui:
            with results_display:
                MemoryResultsDisplay(results, results_display)

    except Exception as e:
        return f"Error executing query: {str(e)}

"

# Add initial query input

online_states = [0] def create_research_interface(Processor):

def helpr(request, session: dict):

    state = {'balance':0, 'research_history': []}
    main_ui = None
    with ui.column().classes("w-full max-w-6xl mx-auto p-6 space-y-6") as loading:
        ui.spinner(size='lg')
        ui.label('Initializing...').classes('ml-2')

    # Container for main content (initially hidden)
    content = ui.column().classes('hidden')

    # Extract session data before spawning thread
    session_id = session.get('ID')
    session_id_h = session.get('IDh')
    session_rid = request.row.query_params.get('session_id') if hasattr(request, 'row') else request.query_params.get('session_id')
    session_valid = session.get('valid')

    # Thread communication
    result_queue = Queue()
    ready_event = Event()

    def init_background():
        nonlocal session_id, session_id_h, session_rid, session_valid
        try:
            # Original initialization logic
            _state, is_new = get_user_state(session_id, is_new=True)

            if is_new and session_id_h != "#0":
                _state = get_user_state(session_id_h)
                save_user_state(session_id, _state)
                delete_user_state(session_id_h)
            if session_rid:
                state_: dict
                state_, is_new_ = get_user_state(session_rid, is_new=True)
                if not is_new_:
                    _state = state_.copy()
                    state_['payment_id'] = ''
                    state_['last_reset'] = datetime.utcnow().isoformat()
                    state_['research_history'] = state_['research_history'][:3]
                    state_['balance'] = 0
                    save_user_state(session_id, _state)
            _state = reset_daily_balance(_state, session_valid)
            save_user_state(session_id, _state)

            # Send result back to main thread
            result_queue.put(_state)
            ready_event.set()
        except Exception as e:
            result_queue.put(e)
            ready_event.set()

        # Start background initialization

    Thread(target=init_background).start()

    def check_ready():
        nonlocal state
        if ready_event.is_set():
            result = result_queue.get()

            # Check if initialization failed
            if isinstance(result, Exception):
                loading.clear()
                with loading:
                    ui.label(f"Error during initialization: {str(result)}").classes('text-red-500')
                return

            # Get state and build main UI
            state = result
            loading.classes('hidden')
            content.classes(remove='hidden')
            main_ui.visible = True
            with main_ui:
                balance.set_text(f"Balance: {state['balance']:.2f}€")
                show_history()
            return  # Stop the timer

        # Check again in 100ms
        ui.timer(0.1, check_ready, once=True)

    # Start checking for completion
    check_ready()

    # Wir speichern die aktive Instanz, damit Follow-Up Fragen gestellt werden können
    processor_instance = {"instance": None}

    # UI-Elemente als Platzhalter; wir definieren sie später in der UI und machen sie so
    # in den Callback-Funktionen über "nonlocal" verfügbar.
    overall_progress = None
    status_label = None
    results_card = None
    summary_content = None
    analysis_content = None
    references_content = None
    followup_card = None
    research_card = None
    config_cart = None
    progress_card = None
    balance = None
    graph_ui = None

    sr_button = None
    r_button = None
    r_text = None


    # Global config storage with default values
    config = {
        'chunk_size': 21000,
        'overlap': 600,
        'num_search_result_per_query': 3,
        'max_search': 3,
        'num_workers': None
    }

    def update_estimates():
        """
        Dummy estimation based on query length and configuration.
        (Replace with your own non-linear formula if needed.)
        """
        query_text = query.value or ""
        query_length = len(query_text)
        # For example: estimated time scales with chunk size and query length.
        estimated_time ,estimated_price = Processor.estimate_processing_metrics(query_length, **config)
        estimated_time *= max(1, online_states[0] * 6)
        if processor_instance["instance"] is not None:
            estimated_price += .25
        if estimated_time < 60:
            time_str = f"~{int(estimated_time)}s"
        elif estimated_time < 3600:
            minutes = estimated_time // 60
            seconds = estimated_time % 60
            time_str = f"~{int(minutes)}m {int(seconds)}s"
        else:
            hours = estimated_time // 3600
            minutes = (estimated_time % 3600) // 60
            time_str = f"~{int(hours)}h {int(minutes)}m"
        with main_ui:
            query_length_label.set_text(f"Total Papers: {config['max_search']*config['num_search_result_per_query']}")
            time_label.set_text(f"Processing Time: {time_str}")
            price_label.set_text(f"Price: {estimated_price:.2f}€")

        return estimated_price

    def on_config_change(event):
        """
        Update the global config based on input changes and recalc estimates.
        """
        try:
            config['chunk_size'] = int(chunk_size_input.value)
        except ValueError:
            pass
        try:
            config['overlap'] = int(overlap_input.value)
            if config['overlap'] > config['chunk_size'] / 4:
                config['overlap'] = int(config['chunk_size'] / 4)
                with main_ui:
                    overlap_input.value = config['overlap']
        except ValueError:
            pass
        try:
            config['num_search_result_per_query'] = int(num_search_result_input.value)
        except ValueError:
            pass
        try:
            config['max_search'] = int(max_search_input.value)
        except ValueError:
            pass
        try:
            config['num_workers'] = int(num_workers_input.value) if num_workers_input.value != 0 else None
        except ValueError:
            config['num_workers'] = None

        update_estimates()

    def on_query_change():
        update_estimates()

    # Callback, der vom Processor (über processor_instance.callback) aufgerufen wird.
    def update_status(data: dict):
        nonlocal overall_progress, status_label
        if not data:
            return
        # Aktualisiere den Fortschrittsbalken und den aktuellen Schritt (wenn vorhanden)
        with main_ui:
            if isinstance(data, dict):
                progress = data.get("progress", 0)
                step = data.get("step", "Processing...")
                overall_progress.value =round( progress ,2) # nicegui.linear_progress erwartet einen Wert zwischen 0 und 1
                status_label.set_text(f"{step} {data.get('info','')}")
            else:
                status_label.set_text(f"{data}")

    def start_search():
        nonlocal balance

        async def helper():
            nonlocal processor_instance, overall_progress, status_label, results_card,                     summary_content, analysis_content,config, references_content, followup_card,sr_button,r_button,r_text

            try:
                if not validate_inputs():
                    with main_ui:
                        state['balance'] += est_price
                        save_user_state(session_id, state)
                        balance.set_text(f"Balance: {state['balance']:.2f}€")
                    return
                reset_interface()
                show_progress_indicators()

                query_text = query.value.strip()
                # Erzeuge das "tools"-Objekt (abhängig von deiner konkreten Implementation)
                tools = get_tools()
                with main_ui:
                    research_card.visible = False
                    config_cart.visible = False
                    config_section.visible = False
                    query.set_value("")
                # Direkt instanziieren: Eine neue ArXivPDFProcessor-Instanz
                if processor_instance["instance"] is not None:
                    processor = processor_instance["instance"]
                    processor.chunk_size = config['chunk_size']
                    processor.overlap = config['overlap']
                    processor.num_search_result_per_query = config['num_search_result_per_query']
                    processor.max_search = config['max_search']
                    processor.num_workers = config['num_workers']
                    papers, insights = await processor.process(query_text)
                else:
                    processor = Processor(query_text, tools=tools, **config)
                # Setze den Callback so, dass Updates in der GUI angezeigt werden
                    processor.callback = update_status
                    processor_instance["instance"] = processor
                    papers, insights = await processor.process()

                update_results({
                    "papers": papers,
                    "insights": insights
                })
                with main_ui:
                    research_card.visible = True
                    config_cart.visible = True
                    show_history()

            except Exception as e:
                import traceback

                with main_ui:
                    update_status({"progress": 0, "step": "Error", "info": str(e)})
                    state['balance'] += est_price
                    save_user_state(session_id, state)
                    balance.set_text(f"Balance: {state['balance']:.2f}€")
                    ui.notify(f"Error {str(e)})", type="negative")
                    research_card.visible = True
                    config_cart.visible = True
                    config_section.visible = True
                print(traceback.format_exc())

        def target():
            get_app().run_a_from_sync(helper, )

        est_price = update_estimates()
        if est_price > state['balance']:
            with main_ui:
                ui.notify(f"Insufficient balance. Need €{est_price:.2f}", type='negative')
        else:
            state['balance'] -= est_price
            save_user_state(session_id, state)
            with main_ui:
                online_states[0] += 1
                balance.set_text(f"Balance: {state['balance']:.2f}€ Running Queries: {online_states[0]}")

            Thread(target=target, daemon=True).start()
            with main_ui:
                online_states[0] -= 1
                balance.set_text(f"Balance: {get_user_state(session_id)['balance']:.2f}€")


    def show_history():
        with config_cart:
            for idx, entry in enumerate(state['research_history']):
                with ui.card().classes("w-full backdrop-blur-lg bg-white/10 p-4"):
                    ui.label(entry['query']).classes('text-sm')
                    ui.button("Open").on_click(lambda _, i=idx: load_history(i))

    def reset():
        nonlocal processor_instance, results_card, followup_card, sr_button, r_button, r_text
        processor_instance["instance"] = None
        show_progress_indicators()
        with main_ui:
            config_cart.visible = False
            config_section.visible = False
            followup_card.visible = False
            results_card.visible = False
            r_button.visible = False
            r_text.set_text("Research Interface")
            sr_button.set_text("Start Research")
        start_search()
    # UI-Aufbau

    with ui.column().classes("w-full max-w-6xl mx-auto p-6 space-y-6") as main_ui:
        balance = ui.label(f"Balance: {state['balance']:.2f}€").classes("text-s font-semibold")

        config_cart = config_cart

        # --- Research Input UI Card ---
        with ui.card().classes("w-full backdrop-blur-lg bg-white/10 p-4") as research_card:
            r_text = ui.label("Research Interface").classes("text-3xl font-bold mb-4")

            # Query input section with auto-updating estimates
            query = ui.input("Research Query",
                                placeholder="Gib hier deine Forschungsfrage ein...",
                                value="")                     .classes("w-full min-h-[100px]")                     .on('change', lambda e: on_query_change()).style("color: var(--text-color)")

            # --- Action Buttons ---
            with ui.row().classes("mt-4"):
                sr_button =ui.button("Start Research", on_click=start_search)                         .classes("bg-blue-600 hover:bg-blue-700 py-3 rounded-lg")
                ui.button("toggle config",
                          on_click=lambda: setattr(config_section, 'visible', not config_section.visible) or show_progress_indicators()).style(
                    "color: var(--text-color)")
                r_button = ui.button("Start new Research",
                          on_click=reset).style(
                    "color: var(--text-color)")
        sr_button = sr_button
        r_button = r_button
        r_button.visible = False
        research_card = research_card

        # --- Options Cart / Configurations ---
        with ui.card_section().classes("w-full backdrop-blur-lg bg-white/10 hidden") as config_section:
            ui.separator()
            ui.label("Configuration Options").classes("text-xl font-semibold mt-4 mb-2")
            with ui.row():
                chunk_size_input = ui.number(label="Chunk Size",
                                             value=config['chunk_size'], format='%.0f', max=64_000, min=1000,
                                             step=100)                         .on('change', on_config_change).style("color: var(--text-color)")
                overlap_input = ui.number(label="Overlap",
                                          value=config['overlap'], format='%.0f', max=6400, min=100, step=50)                         .on('change', on_config_change).style("color: var(--text-color)")

            with ui.row():
                num_search_result_input = ui.number(label="Results per Query",
                                                    value=config['num_search_result_per_query'], format='%.0f',
                                                    min=1, max=100, step=1)                         .on('change', on_config_change).style("color: var(--text-color)")
                max_search_input = ui.number(label="Max Search Queries",
                                             value=config['max_search'], format='%.0f', min=1, max=100, step=1)                         .on('change', on_config_change).style("color: var(--text-color)")
                num_workers_input = ui.number(label="Number of Workers (leave empty for default)",
                                              value=0, format='%.0f', min=0, max=32, step=1)                         .on('change', on_config_change).style("color: var(--text-color)")
        config_section = config_section
        config_section.visible = False
        # --- Ergebnisse anzeigen ---
        with ui.card().classes("w-full backdrop-blur-lg p-4 bg-white/10") as results_card:
            ui.label("Research Results").classes("text-xl font-semibold mb-4")
            with ui.tabs() as tabs:
                ui.tab("Summary")
                ui.tab("References")
                ui.tab("SystemStates")
            with ui.tab_panels(tabs, value="Summary").classes("w-full").style("background-color: var(--background-color)"):
                with ui.tab_panel("Summary"):
                    summary_content = ui.markdown("").style("color : var(--text-color)")
                with ui.tab_panel("References"):
                    references_content = ui.markdown("").style("color : var(--text-color)")
                with ui.tab_panel("SystemStates"):
                    analysis_content = ui.markdown("").style("color : var(--text-color)")


        # Ergebnisse sichtbar machen, sobald sie vorliegen.
        results_card = results_card
        results_card.visible = False

        # --- Follow-Up Bereich mit mehrfachen Folgefragen und Suchparametern ---
        with ui.card().classes("w-full backdrop-blur-lg bg-white/10 p-4 hidden") as followup_card:
            pass

        # Zugriff auf followup_card (falls später benötigt)
        followup_card = followup_card
        followup_card.visible = False

        # --- Fortschrittsanzeige ---
        with ui.card().classes("w-full backdrop-blur-lg bg-white/10 p-4") as progress_card:
            with ui.row():
                ui.label("Research Progress").classes("text-xl font-semibold mb-4")
                query_length_label = ui.label("").classes("mt-6 hover:text-primary transition-colors duration-300")
                time_label = ui.label("Time: ...").classes("mt-6 hover:text-primary transition-colors duration-300")
                price_label = ui.label("Price: ...").classes(
                    "mt-6 hover:text-primary transition-colors duration-300")

            overall_progress = ui.linear_progress(0).classes("w-full mb-4")
            status_label = ui.label("Warte auf Start...").classes("text-base")
        # Wir merken uns progress_card, falls wir ihn zurücksetzen wollen.
        progress_card = progress_card

        query_length_label = query_length_label
        time_label = time_label
        price_label = price_label

        with ui.card().classes("w-full backdrop-blur-lg bg-white/10 p-4") as config_cart:
            # --- Process Code Section ---
            # --- Estimated Time and Price ---
            # ui.label("History").classes("text-xl font-semibold mt-4 mb-2")
            ui.label('Research History').classes('text-xl p-4')
            show_history()

        ui.button('Add Credits', on_click=lambda: balance_overlay(session_id)).props('icon=paid')
        ui.label('About TruthSeeker').classes(
            'mt-6 text-gray-500 hover:text-primary '
            'transition-colors duration-300'
        ).on('click', lambda: ui.navigate.to('/open-Seeker.about', new_tab=True))

        with ui.element('div').classes("w-full").style("white:100%; height:100%") as graph_ui:
            pass

        with ui.card().classes("w-full p-4").style("background-color: var(--background-color)"):
            ui.label("Private Session link (restore the session on a different device)")
            base_url = f'https://{os.getenv("HOSTNAME")}/gui/open-Seeker.seek' if not 'localhost' in os.getenv("HOSTNAME") else 'http://localhost:5000/gui/open-Seeker.seek'
            ui.label(f"{base_url}?session_id={session_id}").style("white:100%")
            ui.label("Changes each time!")

        graph_ui = graph_ui
        graph_ui.visible = False
    main_ui = main_ui
    main_ui.visible = False

    # --- Hilfsfunktionen ---
    def validate_inputs() -> bool:
        if not query.value.strip():
            with main_ui:
                ui.notify("Bitte gib eine Forschungsfrage ein.", type="warning")
            return False
        return True

    def reset_interface():
        nonlocal overall_progress, status_label, results_card, followup_card
        overall_progress.value = 0
        with main_ui:
            status_label.set_text("Research startet...")
        # Ergebnisse und Follow-Up Bereich verstecken
        results_card.visible = False
        followup_card.visible = False
        graph_ui.visible = False

    def show_progress_indicators():
        nonlocal progress_card
        progress_card.visible = True

    def update_results(data: dict, save=True):
        nonlocal summary_content, analysis_content, references_content, results_card,                followup_card,graph_ui, r_button, r_text, sr_button
        with main_ui:
            r_button.visible = True
            r_text.set_text("Add to current Results or press 'Start new Research'")
            sr_button.set_text("Add to current Results")
        # Handle papers (1-to-1 case)
        papers = data.get("papers", [])
        if not isinstance(papers, list):
            papers = [papers]

        # Get insights
        insights = data.get("insights", [])

        if save:
            history_entry = data.copy()
            history_entry['papers'] = [paper.model_dump_json() for paper in papers]
            if processor_instance is not None and processor_instance['instance'] is not None:
                history_entry["mam_name"] = processor_instance['instance'].mem_name
                history_entry["query"] = processor_instance['instance'].query

                history_entry["processor_memory"] = processor_instance['instance'].tools.get_memory(

                ).save_memory(history_entry["mam_name"], None)
            state['research_history'].append(history_entry)
            save_user_state(session_id, state)
        else:
            papers = [Paper(**json.loads(paper)) for paper in papers]
        create_followup_section(processor_instance, followup_card, session_id, balance)
        with main_ui:
            progress_card.visible = False
            # Build summary from insights
            summaries = []
            for insight in insights:
                if 'result' in insight and 'summary' in insight['result']:
                    if isinstance(insight['result']['summary'], str):
                        # print(insight['result']['summary'], "NEXT", json.loads(insight['result']['summary'][:-1]),"NEXT22",  type(json.loads(insight['result']['summary'][:-1])))
                        insight['result']['summary'] = json.loads(insight['result']['summary'][:-1])
                    main_summary = insight['result']['summary'].get('main_summary', '')
                    if main_summary:
                        summaries.append(main_summary)
            summary_text = "

".join(summaries) if summaries else "No summary available." summary_content.set_content(f"# Research Summary

{summary_text}")

            # Analysis section (unchanged if processor details haven't changed)
            if processor_instance["instance"] is not None:
                inst = processor_instance["instance"]
                analysis_md = (
                    f"# Analysis

" f"- query: {inst.query} " f"- chunk_size: {inst.chunk_size} " f"- overlap: {inst.overlap} " f"- max_workers: {inst.max_workers} " f"- num_search_result_per_query: {inst.nsrpq} " f"- max_search: {inst.max_search} " f"- download_dir: {inst.download_dir} " f"- mem_name: {inst.mem_name} " f"- current_session: {inst.current_session} " f"- all_ref_papers: {inst.all_ref_papers} " f"- all_texts_len: {inst.all_texts_len} " f"- final_texts_len: {inst.f_texts_len} " f"- num_workers: {inst.num_workers}" ) analysis_content.set_content(analysis_md)

            # References and Insights section
            references_md = "# References

" # Add papers references_md += " ".join( f"- ({i}) {getattr(paper, 'title', 'Unknown Title')}})" for i, paper in enumerate(papers) )

            # Add detailed insights
            references_md += "

Insights

" for i, insight in enumerate(insights): print(insight) result = insight.get('result', {}) summary = result.get('summary', {})

                if isinstance(summary, str):
                    summary = json.loads(summary)

                # Main summary
                references_md += f"

Insight

" references_md += f"### Main Summary {summary.get('main_summary', 'No summary available.')} "

                # Concept Analysis
                concept_analysis = summary.get('concept_analysis', {})
                if concept_analysis:
                    references_md += "

Concept Analysis

" references_md += "#### Key Concepts - " + " - ".join( concept_analysis.get('key_concepts', [])) + " " references_md += "

Relationships

• " + "
• ".join( concept_analysis.get('relationships', [])) + " " references_md += "

Importance Hierarchy

• " + "

• ".join( concept_analysis.get('importance_hierarchy', [])) + " "

              # Topic Insights
              topic_insights = summary.get('topic_insights', {})
              if topic_insights:
                  references_md += "
  

  Topic Insights

  " references_md += "#### Primary Topics - " + " - ".join( topic_insights.get('primary_topics', [])) + " " references_md += "

  Cross References

  • " + "
  • ".join( topic_insights.get('cross_references', [])) + " " references_md += "

  Knowledge Gaps

  • " + "

  • ".join( topic_insights.get('knowledge_gaps', [])) + " "

            # Relevance Assessment
            relevance = summary.get('relevance_assessment', {})
            if relevance:
                references_md += "
    

    Relevance Assessment

    " references_md += f"- Query Alignment: {relevance.get('query_alignment', 'N/A')} " references_md += f"- Confidence Score: {relevance.get('confidence_score', 'N/A')} " references_md += f"- Coverage Analysis: {relevance.get('coverage_analysis', 'N/A')} "

        references_content.set_content(references_md)
    
        # nx concpts graph
        if processor_instance["instance"] is not None:
            create_graph_tab(
                processor_instance,
                graph_ui,main_ui
            )
    
        # Show results and followup cards
        results_card.visible = True
        followup_card.visible = True
        graph_ui.visible = True
    

    def load_history(index: int): entry = state['research_history'][index] if processor_instance is not None and processor_instance['instance'] is not None:

        processor_instance["instance"].mem_name = entry["mam_name"]
        processor_instance['instance'].query = entry["query"]
    
        pass
    else:
        processor = Processor(entry["query"], tools=get_tools(), **config)
        # Setze den Callback so, dass Updates in der GUI angezeigt werden
        processor.callback = update_status
        processor.mem_name = entry["mam_name"]
        processor_instance["instance"] = processor
    
    processor_instance["instance"].tools.get_memory().load_memory(entry["mam_name"], entry["processor_memory"])
    processor_instance["instance"].mem_name = entry["mam_name"]
    update_results(entry, save=False)
    

  return helpr

--- Stripe Integration ---

def regiser_stripe_integration(is_scc=True): def stripe_callback(request: Request):

    sid = request.row.query_params.get('session_id') if hasattr(request, 'row') else request.query_params.get('session_id')
    state = get_user_state(sid)

    if state['payment_id'] == '':
        with ui.card().classes("w-full items-center").style("background-color: var(--background-color)"):
            ui.label(f"No payment id!").classes("text-lg font-bold")
            ui.button(
                "Start Research",
                on_click=lambda: ui.navigate.to("/open-Seeker.seek?session_id="+sid)
            ).classes(
                "w-full px-6 py-4 text-lg font-bold "
                "bg-primary hover:bg-primary-dark "
                "transform hover:-translate-y-0.5 "
                "transition-all duration-300 ease-in-out "
                "rounded-xl shadow-lg animate-slideUp"
            )
        return

    try:
        session_data = stripe.checkout.Session.retrieve(state['payment_id'])
    except Exception as e:
        with ui.card().classes("w-full items-center").style("background-color: var(--background-color)"):
            ui.label(f"No Transactions Details !{e}").classes("text-lg font-bold")
            ui.button(
                "Start Research",
                on_click=lambda: ui.navigate.to("/open-Seeker.seek")
            ).classes(
                "w-full px-6 py-4 text-lg font-bold "
                "bg-primary hover:bg-primary-dark "
                "transform hover:-translate-y-0.5 "
                "transition-all duration-300 ease-in-out "
                "rounded-xl shadow-lg animate-slideUp"
            )
            return
    with ui.card().classes("w-full items-center").style("background-color: var(--background-color)"):
        if is_scc and state['payment_id'] != '' and session_data.payment_status == 'paid':
            state = get_user_state(sid)
            amount = session_data.amount_total / 100  # Convert cents to euros
            state['balance'] += amount
            state['payment_id'] = ''
            save_user_state(sid, state)

        # ui.navigate.to(f'/session?session={session}')
            ui.label(f"Transaction Complete - New balance :{state['balance']}").classes("text-lg font-bold")
            with ui.card().classes("w-full p-4").style("background-color: var(--background-color)"):
                ui.label("Private Session link (restore the session on a different device)")
                base_url = f'https://{os.getenv("HOSTNAME")}/gui/open-Seeker.seek' if not 'localhost' in os.getenv("HOSTNAME")else 'http://localhost:5000/gui/open-Seeker.seek'
                ui.label(f"{base_url}?session_id={sid}").style("white:100%")
                ui.label("Changes each time!")
        else:
            ui.label(f"Transaction Error! {session_data}, {dir(session_data)}").classes("text-lg font-bold")
        ui.button(
            "Start Research",
            on_click=lambda: ui.navigate.to("/open-Seeker.seek")
        ).classes(
            "w-full px-6 py-4 text-lg font-bold "
            "bg-primary hover:bg-primary-dark "
            "transform hover:-translate-y-0.5 "
            "transition-all duration-300 ease-in-out "
            "rounded-xl shadow-lg animate-slideUp"
        )


return stripe_callback

def handle_stripe_payment(amount: float, session_id): base_url = f'https://{os.getenv("HOSTNAME")}/gui/open-Seeker.stripe' if not 'localhost' in os.getenv("HOSTNAME") else 'http://localhost:5000/gui/open-Seeker.stripe' session = stripe.checkout.Session.create( payment_method_types=['card', "link", ], line_items=[{ 'price_data': { 'currency': 'eur', 'product_data': {'name': 'Research Credits'}, 'unit_amount': int(amount * 100), }, 'quantity': 1, }], automatic_tax={"enabled": True}, mode='payment', success_url=f'{base_url}?session_id={session_id}', cancel_url=f'{base_url}.error' ) state = get_user_state(session_id) state['payment_id'] = session.id save_user_state(session_id, state) ui.navigate.to(session.url, new_tab=True)

--- UI Components ---

def balance_overlay(session_id): with ui.dialog().classes('w-full max-w-md bg-white/20 backdrop-blur-lg rounded-xl') as dialog: with ui.card().classes('w-full p-6 space-y-4').style("background-color: var(--background-color)"): ui.label('Add Research Credits').classes('text-2xl font-bold') amount = ui.number('Amount (€) min 2', value=5, format='%.2f', min=2, max=9999, step=1).classes('w-full') with ui.row().classes('w-full justify-between'): ui.button('Cancel', on_click=dialog.close).props('flat') ui.button('Purchase', on_click=lambda: handle_stripe_payment(amount.value, session_id)) return dialog

def create_ui(processor): # ui_instance = register_nicegui("open-Seeker", create_landing_page , additional=""" """, show=False) register_nicegui("open-Seeker.demo", create_video_demo, additional=""" """, show=False)

newui

cleanup_module(app)

Cleanup resources when the module is unloaded

Source code in toolboxv2/mods/TruthSeeker/newui.py

@export(mod_name=MOD_NAME, version=version, exit_f=True)
def cleanup_module(app: App):
    """Cleanup resources when the module is unloaded"""
    # Clean up any temp files or resources
    import glob
    import shutil

    # Remove temporary PDF directories
    for pdf_dir in glob.glob("pdfs_*"):
        try:
            shutil.rmtree(pdf_dir)
        except Exception as e:
            print(f"Error removing directory {pdf_dir}: {str(e)}")

    # Clear any SSE queues
    if hasattr(app, 'sse_queues'):
        app.sse_queues = {}

    if hasattr(app, 'payment_queues'):
        app.payment_queues = {}

    return Result.ok(info="ArXivPDFProcessor UI cleaned up")

create_payment(app, data) async

Create a Stripe payment session

Source code in toolboxv2/mods/TruthSeeker/newui.py

@export(mod_name=MOD_NAME, api=True, version=version)
async def create_payment(app: App, data):
    """Create a Stripe payment session"""
    amount = data.get("amount")
    session_id = data.get("session_id")

    if amount < 2:
        return Result.default_user_error(info="Minimum donation amount is €2")

    try:
        # Create a Stripe Checkout Session
        base_url = f"https://{os.getenv('HOSTNAME', 'localhost:5000')}"
        success_url = f"{base_url}/api/{MOD_NAME}/payment_success?session_id={session_id}"
        cancel_url = f"{base_url}/api/{MOD_NAME}/payment_cancel?session_id={session_id}"
        stripe = __import__('stripe')
        stripe.api_key = os.getenv('STRIPE_SECRET_KEY', 'sk_test_YourSecretKey')

        stripe_session = stripe.checkout.Session.create(
            payment_method_types=['card', 'link'],
            line_items=[{
                'price_data': {
                    'currency': 'eur',
                    'product_data': {'name': 'Research Credits'},
                    'unit_amount': int(amount * 100),
                },
                'quantity': 1,
            }],
            automatic_tax={"enabled": True},
            mode='payment',
            success_url=success_url,
            cancel_url=cancel_url
        )

        # Store the payment info
        if not hasattr(app, 'payment_info'):
            app.payment_info = {}

        # Initialize payment_queues if not already done
        if not hasattr(app, 'payment_queues'):
            app.payment_queues = {}

        # Create a queue for this payment
        app.payment_queues[session_id] = asyncio.Queue()

        app.payment_info[session_id] = {
            'payment_id': stripe_session.id,
            'amount': amount,
            'status': 'pending'
        }

        return Result.ok(data={"url": stripe_session.url})
    except Exception as e:
        return Result.default_internal_error(info=f"Error creating payment: {str(e)}")

estimate_processing(data) async

Estimate processing time and cost for a given query

Source code in toolboxv2/mods/TruthSeeker/newui.py

@export(mod_name=MOD_NAME, api=True, version=version)
async def estimate_processing(data):
    """Estimate processing time and cost for a given query"""
    # Use the static method to estimate metrics
    query, max_search, num_search_result_per_query= data.get("query", ""), data.get("max_search",4), data.get("num_search_result_per_query",6)
    estimated_time, estimated_price = ArXivPDFProcessor.estimate_processing_metrics(
        query_length=len(query),
        max_search=max_search,
        num_search_result_per_query=num_search_result_per_query,
        chunk_size=1_000_000,
        overlap=2_000,
        num_workers=None
    )

    return Result.ok(data={
        "time": estimated_time,
        "price": estimated_price
    })

follow_up_query(app, data) async

Ask a follow-up question about the research

Source code in toolboxv2/mods/TruthSeeker/newui.py

@export(mod_name=MOD_NAME, api=True, version=version)
async def follow_up_query(app: App, data):
    """Ask a follow-up question about the research"""
    research_id = data.get("research_id")
    query = data.get("query")

    if not hasattr(app, 'research_processes') or research_id not in app.research_processes:
        return Result.default_user_error(info="Research process not found")

    research_process = app.research_processes[research_id]

    if research_process['status'] != 'complete':
        return Result.default_user_error(info="Research is not complete")

    processor = research_process['processor']
    if not processor:
        return Result.default_user_error(info="Processor not available")

    try:
        # Use the extra_query method to ask follow-up questions
        result = await processor.extra_query(query)

        return Result.ok(data={"answer": result['response'] if result and 'response' in result else "No response"})
    except Exception as e:
        return Result.default_internal_error(info=f"Error processing follow-up query: {str(e)}")

initialize_module(app)

Initialize the module and register UI with CloudM

Source code in toolboxv2/mods/TruthSeeker/newui.py

@export(mod_name=MOD_NAME, version=version, initial=True)
def initialize_module(app: App):
    """Initialize the module and register UI with CloudM"""
    # Register the UI with CloudM
    app.run_any(("CloudM", "add_ui"),
                name="TruthSeeker",
                title="TruthSeeker Research",
                path=f"/api/{MOD_NAME}/get_main_ui",
                description="AI Research Assistant"
                )

    # Initialize SSE message queues
    if not hasattr(app, 'sse_queues'):
        app.sse_queues = {}
    print("TruthSeeker online")
    return Result.ok(info="ArXivPDFProcessor UI initialized")

payment_cancel(app, session_id, request_as_kwarg=True, request=None) async

Handle cancelled payment

Source code in toolboxv2/mods/TruthSeeker/newui.py

@export(mod_name=MOD_NAME, api=True, version=version)
async def payment_cancel(app: App, session_id: str, request_as_kwarg=True, request=None):
    """Handle cancelled payment"""
    if hasattr(app, 'payment_info') and session_id in app.payment_info:
        app.payment_info[session_id]['status'] = 'cancelled'

        # Notify SSE clients about payment cancellation
        if hasattr(app, 'payment_queues') and session_id in app.payment_queues:
            await app.payment_queues[session_id].put({
                "status": "cancelled"
            })

    return Result.html(app.web_context() + """
    <div style="text-align: center; padding: 50px;">
        <h2>Payment Cancelled</h2>
        <p>Your payment was cancelled.</p>
        <script>
            setTimeout(function() {
                window.close();
            }, 3000);
        </script>
    </div>
    """)

payment_stream(app, session_id) async

SSE stream endpoint for payment status updates

Source code in toolboxv2/mods/TruthSeeker/newui.py

@export(mod_name=MOD_NAME, api=True, version=version)
async def payment_stream(app: App, session_id: str):
    """SSE stream endpoint for payment status updates"""
    if not hasattr(app, 'payment_queues'):
        app.payment_queues = {}

    # Create a message queue for this session_id if it doesn't exist
    if session_id not in app.payment_queues:
        app.payment_queues[session_id] = asyncio.Queue()

    async def generate():
        try:
            # Stream payment updates
            while True:
                try:
                    # Wait for a payment update with a timeout
                    payment_data = await asyncio.wait_for(app.payment_queues[session_id].get(), timeout=30)
                    yield f"event: payment_update\ndata: {json.dumps(payment_data)}\n\n"

                    # If the payment is complete or cancelled, exit the loop
                    if payment_data.get('status') in ['completed', 'cancelled']:
                        break
                except TimeoutError:
                    # Send a keep-alive comment to prevent connection timeout
                    yield ":\n\n"
        finally:
            # Clean up resources when the client disconnects
            if session_id in app.payment_queues:
                # Keep the queue for other potential clients
                pass

    return Result.stream(generate())

payment_success(app, session_id, request_as_kwarg=True, request=None) async

Handle successful payment

Source code in toolboxv2/mods/TruthSeeker/newui.py

@export(mod_name=MOD_NAME, api=True, version=version)
async def payment_success(app: App, session_id: str, request_as_kwarg=True, request=None):
    """Handle successful payment"""
    if not hasattr(app, 'payment_info') or session_id not in app.payment_info:
        return Result.html(app.web_context() + """
        <div style="text-align: center; padding: 50px;">
            <h2>Payment Session Not Found</h2>
            <p>Return to the main page to continue.</p>
            <a href="/" style="display: inline-block; margin-top: 20px; padding: 10px 20px; background-color: #4F46E5; color: white; text-decoration: none; border-radius: 5px;">Return to Home</a>
        </div>
        """)

    payment_info = app.payment_info[session_id]

    try:
        # Verify the payment with Stripe
        stripe = __import__('stripe')
        stripe.api_key = os.getenv('STRIPE_SECRET_KEY', 'sk_test_YourSecretKey')

        stripe_session = stripe.checkout.Session.retrieve(payment_info['payment_id'])

        if stripe_session.payment_status == 'paid':
            payment_info['status'] = 'completed'

            # Notify SSE clients about payment completion
            if hasattr(app, 'payment_queues') and session_id in app.payment_queues:
                await app.payment_queues[session_id].put({
                    "status": "completed",
                    "amount": payment_info['amount']
                })

            return Result.html(app.web_context() + """
            <div style="text-align: center; padding: 50px;">
                <h2>Thank You for Your Support!</h2>
                <p>Your payment was successful. You can now close this window and continue with your research.</p>
                <script>
                    setTimeout(function() {
                        window.close();
                    }, 5000);
                </script>
            </div>
            """)
        else:
            return Result.html(app.web_context() + """
            <div style="text-align: center; padding: 50px;">
                <h2>Payment Not Completed</h2>
                <p>Your payment has not been completed. Please try again.</p>
                <button onclick="window.close()">Close Window</button>
            </div>
            """)
    except Exception as e:
        return Result.html(app.web_context() + f"""
        <div style="text-align: center; padding: 50px;">
            <h2>Error Processing Payment</h2>
            <p>There was an error processing your payment: {str(e)}</p>
            <button onclick="window.close()">Close Window</button>
        </div>
        """)

research_results(app, research_id) async

Get the results of a completed research process

Source code in toolboxv2/mods/TruthSeeker/newui.py

@export(mod_name=MOD_NAME, api=True, version=version)
async def research_results(app: App, research_id: str):
    """Get the results of a completed research process"""
    if not hasattr(app, 'research_processes') or research_id not in app.research_processes:
        return Result.default_user_error(info="Research process not found")

    research_process = app.research_processes[research_id]

    if research_process['status'] != 'complete':
        return Result.default_user_error(info="Research is not complete")

    return Result.ok(data=research_process['results'])

research_status(app, research_id) async

Get the status of a research process

Source code in toolboxv2/mods/TruthSeeker/newui.py

@export(mod_name=MOD_NAME, api=True, version=version)
async def research_status(app: App, research_id: str):
    """Get the status of a research process"""
    if not hasattr(app, 'research_processes') or research_id not in app.research_processes:
        return Result.default_user_error(info="Research process not found")

    research_process = app.research_processes[research_id]

    return Result.ok(data={
        "status": research_process['status'],
        "progress": research_process['progress'],
        "step": research_process['step'],
        "info": research_process['info']
    })

start_research(app, data) async

Start a new research process

Source code in toolboxv2/mods/TruthSeeker/newui.py

@export(mod_name=MOD_NAME, api=True, version=version)
async def start_research(app: App, data):
    """Start a new research process"""
    # Get data from the request
    query = data.get("query")
    session_id = data.get("session_id")
    max_search = data.get("max_search", 4)
    num_search_result_per_query = data.get("num_search_result_per_query", 4)

    # Get the tools module
    tools = get_app("ArXivPDFProcessor").get_mod("isaa")
    if not hasattr(tools, 'initialized') or not tools.initialized:
        tools.init_isaa(build=True)

    # Generate a unique research_id
    research_id = str(uuid.uuid4())

    # Store the research information in a global dictionary
    if not hasattr(app, 'research_processes'):
        app.research_processes = {}

    # Initialize SSE queues if not already done
    if not hasattr(app, 'sse_queues'):
        app.sse_queues = {}

    # Create a queue for this research process
    app.sse_queues[research_id] = asyncio.Queue()

    # Create a processor with callback for status updates
    app.research_processes[research_id] = {
        'status': 'initializing',
        'progress': 0.0,
        'step': 'Initializing',
        'info': '',
        'query': query,
        'session_id': session_id,
        'processor': None,
        'results': None,
        'stop_requested': False
    }

    # Define the callback function that sends updates to the SSE queue
    def status_callback(status_data):
        if research_id in app.research_processes:
            process = app.research_processes[research_id]
            process['status'] = 'processing'
            process['progress'] = status_data.get('progress', 0.0)
            process['step'] = status_data.get('step', '')
            process['info'] = status_data.get('info', '')

            # Put the status update in the SSE queue
            status_update = {
                "status": process['status'],
                "progress": process['progress'],
                "step": process['step'],
                "info": process['info']
            }

            if research_id in app.sse_queues:
                asyncio.create_task(app.sse_queues[research_id].put(status_update))

    # Create the processor
    processor = ArXivPDFProcessor(
        query=query,
        tools=tools,
        chunk_size=1_000_000,
        overlap=2_000,
        max_search=max_search,
        num_search_result_per_query=num_search_result_per_query,
        download_dir=f"pdfs_{research_id}",
        callback=status_callback
    )

    app.research_processes[research_id]['processor'] = processor

    # Process in the background
    async def process_in_background():
        try:
            # Check if stop was requested before starting
            if app.research_processes[research_id]['stop_requested']:
                app.research_processes[research_id]['status'] = 'stopped'
                if research_id in app.sse_queues:
                    await app.sse_queues[research_id].put({
                        "status": "stopped",
                        "progress": 0,
                        "step": "Research stopped",
                        "info": ""
                    })
                return

            # Start processing
            papers, insights = await processor.process()

            # Check if stop was requested during processing
            if app.research_processes[research_id]['stop_requested']:
                app.research_processes[research_id]['status'] = 'stopped'
                if research_id in app.sse_queues:
                    await app.sse_queues[research_id].put({
                        "status": "stopped",
                        "progress": 1,
                        "step": "Research stopped",
                        "info": ""
                    })
                return

            # Store results
            app.research_processes[research_id]['results'] = {
                'papers': papers,
                'insights': insights['response'] if insights and 'response' in insights else None
            }
            app.research_processes[research_id]['status'] = 'complete'

            # Send final status update
            if research_id in app.sse_queues:
                await app.sse_queues[research_id].put({
                    "status": "complete",
                    "progress": 1,
                    "step": "Research complete",
                    "info": f"Found {len(papers)} papers"
                })

        except Exception as e:
            app.research_processes[research_id]['status'] = 'error'
            app.research_processes[research_id]['info'] = str(e)

            # Send error status
            if research_id in app.sse_queues:
                await app.sse_queues[research_id].put({
                    "status": "error",
                    "progress": 0,
                    "step": "Error",
                    "info": str(e)
                })

            print(f"Error in research process {research_id}: {str(e)}")

    # Start the background task
    asyncio.create_task(process_in_background())

    return Result.ok(data={"research_id": research_id})

status_stream(app, research_id) async

SSE stream endpoint for research status updates

Source code in toolboxv2/mods/TruthSeeker/newui.py

@export(mod_name=MOD_NAME, api=True, version=version)
async def status_stream(app: App, research_id: str):
    """SSE stream endpoint for research status updates"""
    if not hasattr(app, 'sse_queues'):
        app.sse_queues = {}

    # Create a message queue for this research_id if it doesn't exist
    if research_id not in app.sse_queues:
        app.sse_queues[research_id] = asyncio.Queue()

    async def generate():
        # Send initial status
        if hasattr(app, 'research_processes') and research_id in app.research_processes:
            process = app.research_processes[research_id]
            initial_status = {
                "status": process['status'],
                "progress": process['progress'],
                "step": process['step'],
                "info": process['info']
            }
            yield f"event: status_update\ndata: {json.dumps(initial_status)}\n\n"

        try:
            # Stream status updates
            while True:
                try:
                    # Wait for a new status update with a timeout
                    status_data = await asyncio.wait_for(app.sse_queues[research_id].get(), timeout=30)
                    yield f"event: status_update\ndata: {json.dumps(status_data)}\n\n"

                    # If the research is complete or there was an error, exit the loop
                    if status_data.get('status') in ['complete', 'error', 'stopped']:
                        break
                except TimeoutError:
                    # Send a keep-alive comment to prevent connection timeout
                    yield ":\n\n"
        finally:
            # Clean up resources when the client disconnects
            if research_id in app.sse_queues:
                # Keep the queue for other potential clients
                pass

    return Result.stream(generate())

stop_research(app, data) async

Stop a research process

Source code in toolboxv2/mods/TruthSeeker/newui.py

@export(mod_name=MOD_NAME, api=True, version=version)
async def stop_research(app: App, data):
    """Stop a research process"""
    research_id = data.get("research_id")
    if not hasattr(app, 'research_processes') or research_id not in app.research_processes:
        return Result.default_user_error(info="Research process not found")

    app.research_processes[research_id]['stop_requested'] = True

    # Send stopped status to SSE clients
    if hasattr(app, 'sse_queues') and research_id in app.sse_queues:
        await app.sse_queues[research_id].put({
            "status": "stopped",
            "progress": app.research_processes[research_id]['progress'],
            "step": "Stopping research",
            "info": ""
        })

    return Result.ok(data={"status": "stop_requested"})

tests

TestTruthSeeker

Bases: TestCase

Source code in toolboxv2/mods/TruthSeeker/tests.py

class TestTruthSeeker(unittest.TestCase):
    def setUp(self):
        # Mock the App class
        self.mock_app = Mock()
        self.mock_app.get_mod.return_value = Mock()

        # Setup mock for run_any that returns iterable dict
        self.mock_app.run_any.return_value = {
            "1": {"name": "template1"},
            "2": {"name": "template2"}
        }

        # Mock RequestSession
        self.mock_request = Mock()
        self.mock_request.json = AsyncMock()

    @patch('os.path.join')
    @patch('builtins.open', create=True)
    def test_start_initialization(self, mock_open, mock_join):
        """Test the start function initializes correctly"""
        # Setup mock file handling
        mock_file = Mock()
        mock_file.read.return_value = "test content"
        mock_open.return_value.__enter__.return_value = mock_file

        # Call start function
        start(self.mock_app)

        # Verify app initialization calls
        self.mock_app.get_mod.assert_called_with("CodeVerification")
        self.mock_app.run_any.assert_any_call(("CodeVerification", "init_scope"), scope="TruthSeeker")
        self.mock_app.run_any.assert_any_call(("CodeVerification", "init_scope"), scope="TruthSeeker-promo")

    @async_test
    async def test_codes_valid_request(self):
        """Test the codes function with valid input"""
        # Mock request data
        test_data = {
            "query": "test query",
            "depth": "Q",
            "promoCode": "PROMO15",
            "ontimeCode": "TEST123"
        }
        self.mock_request.json.return_value = test_data

        # Mock code verification
        self.mock_app.run_any.return_value = {
            "template_name": "Promo15",
            "usage_type": "one_time"
        }

        result = await codes(self.mock_app, self.mock_request)

        self.assertTrue(result['valid'])
        self.assertIn('ontimeKey', result)
        self.assertIn('ppc', result)

    @async_test
    async def test_codes_invalid_promo(self):
        """Test the codes function with invalid promo code"""
        test_data = {
            "query": "test query",
            "depth": "I",
            "promoCode": "INVALID",
            "ontimeCode": "TEST123"
        }
        self.mock_request.json.return_value = test_data

        # Mock invalid promo code verification
        self.mock_app.run_any.return_value = None

        result = await codes(self.mock_app, self.mock_request)

        self.assertIn('ppc', result)
        self.assertTrue(result['ppc']['price'] > 0)

    @async_test
    async def test_process_valid_request(self):
        """Test the process function with valid input"""
        test_data = {
            "query": "test query",
            "depth": "Q",
            "ontimeKey": "VALID_KEY",
            "email": "test@example.com"
        }
        self.mock_request.json.return_value = test_data

        # Mock valid key verification
        self.mock_app.run_any.return_value = {
            "template_name": "PROCESS",
            "usage_type": "timed",
            "uses_count": 1
        }

        # Mock ArXivPDFProcessor
        with patch('toolboxv2.mods.TruthSeeker.module.ArXivPDFProcessor') as mock_processor:
            mock_insights = MagicMock()
            mock_insights.is_true = "True"
            mock_insights.summary = "Test summary"
            mock_insights.key_point = "Point1>\n\n<Point2"

            mock_processor.return_value.process.return_value = ([], mock_insights)

            result = await process(self.mock_app, self.mock_request)

            self.assertEqual(result['is_true'], "True")
            self.assertEqual(result['summary'], "Test summary")

    @async_test
    async def test_process_invalid_key(self):
        """Test the process function with invalid key"""
        test_data = {
            "query": "test query",
            "depth": "Q",
            "ontimeKey": "INVALID_KEY",
            "email": "test@example.com"
        }
        self.mock_request.json.return_value = test_data

        # Mock invalid key verification
        self.mock_app.run_any.return_value = None

        result = await process(self.mock_app, self.mock_request)

        self.assertEqual(result['summary'], "INVALID QUERY")
        self.assertEqual(result['insights'], [])
        self.assertEqual(result['papers'], [])

    def test_byCode_functionality(self):
        """Test the byCode function"""
        test_request = Mock()
        test_request.json.return_value = ["payKey", "codeClass", "ontimeKey"]

        result = byCode(self.mock_app, test_request)

        self.assertEqual(result, {'code': 'code'})

test_byCode_functionality()

Test the byCode function

Source code in toolboxv2/mods/TruthSeeker/tests.py

def test_byCode_functionality(self):
    """Test the byCode function"""
    test_request = Mock()
    test_request.json.return_value = ["payKey", "codeClass", "ontimeKey"]

    result = byCode(self.mock_app, test_request)

    self.assertEqual(result, {'code': 'code'})

test_codes_invalid_promo() async

Test the codes function with invalid promo code

Source code in toolboxv2/mods/TruthSeeker/tests.py

@async_test
async def test_codes_invalid_promo(self):
    """Test the codes function with invalid promo code"""
    test_data = {
        "query": "test query",
        "depth": "I",
        "promoCode": "INVALID",
        "ontimeCode": "TEST123"
    }
    self.mock_request.json.return_value = test_data

    # Mock invalid promo code verification
    self.mock_app.run_any.return_value = None

    result = await codes(self.mock_app, self.mock_request)

    self.assertIn('ppc', result)
    self.assertTrue(result['ppc']['price'] > 0)

test_codes_valid_request() async

Test the codes function with valid input

Source code in toolboxv2/mods/TruthSeeker/tests.py

@async_test
async def test_codes_valid_request(self):
    """Test the codes function with valid input"""
    # Mock request data
    test_data = {
        "query": "test query",
        "depth": "Q",
        "promoCode": "PROMO15",
        "ontimeCode": "TEST123"
    }
    self.mock_request.json.return_value = test_data

    # Mock code verification
    self.mock_app.run_any.return_value = {
        "template_name": "Promo15",
        "usage_type": "one_time"
    }

    result = await codes(self.mock_app, self.mock_request)

    self.assertTrue(result['valid'])
    self.assertIn('ontimeKey', result)
    self.assertIn('ppc', result)

test_process_invalid_key() async

Test the process function with invalid key

Source code in toolboxv2/mods/TruthSeeker/tests.py

@async_test
async def test_process_invalid_key(self):
    """Test the process function with invalid key"""
    test_data = {
        "query": "test query",
        "depth": "Q",
        "ontimeKey": "INVALID_KEY",
        "email": "test@example.com"
    }
    self.mock_request.json.return_value = test_data

    # Mock invalid key verification
    self.mock_app.run_any.return_value = None

    result = await process(self.mock_app, self.mock_request)

    self.assertEqual(result['summary'], "INVALID QUERY")
    self.assertEqual(result['insights'], [])
    self.assertEqual(result['papers'], [])

test_process_valid_request() async

Test the process function with valid input

Source code in toolboxv2/mods/TruthSeeker/tests.py

@async_test
async def test_process_valid_request(self):
    """Test the process function with valid input"""
    test_data = {
        "query": "test query",
        "depth": "Q",
        "ontimeKey": "VALID_KEY",
        "email": "test@example.com"
    }
    self.mock_request.json.return_value = test_data

    # Mock valid key verification
    self.mock_app.run_any.return_value = {
        "template_name": "PROCESS",
        "usage_type": "timed",
        "uses_count": 1
    }

    # Mock ArXivPDFProcessor
    with patch('toolboxv2.mods.TruthSeeker.module.ArXivPDFProcessor') as mock_processor:
        mock_insights = MagicMock()
        mock_insights.is_true = "True"
        mock_insights.summary = "Test summary"
        mock_insights.key_point = "Point1>\n\n<Point2"

        mock_processor.return_value.process.return_value = ([], mock_insights)

        result = await process(self.mock_app, self.mock_request)

        self.assertEqual(result['is_true'], "True")
        self.assertEqual(result['summary'], "Test summary")

test_start_initialization(mock_open, mock_join)

Test the start function initializes correctly

Source code in toolboxv2/mods/TruthSeeker/tests.py

@patch('os.path.join')
@patch('builtins.open', create=True)
def test_start_initialization(self, mock_open, mock_join):
    """Test the start function initializes correctly"""
    # Setup mock file handling
    mock_file = Mock()
    mock_file.read.return_value = "test content"
    mock_open.return_value.__enter__.return_value = mock_file

    # Call start function
    start(self.mock_app)

    # Verify app initialization calls
    self.mock_app.get_mod.assert_called_with("CodeVerification")
    self.mock_app.run_any.assert_any_call(("CodeVerification", "init_scope"), scope="TruthSeeker")
    self.mock_app.run_any.assert_any_call(("CodeVerification", "init_scope"), scope="TruthSeeker-promo")

run_all_tests()

Run all test classes

Source code in toolboxv2/mods/TruthSeeker/tests.py

@default_test
def run_all_tests():
    """Run all test classes"""
    return run_test_suite()

run_arxiv_processor_tests(test_name=None)

Run TestArXivPDFProcessor tests

Source code in toolboxv2/mods/TruthSeeker/tests.py

def run_arxiv_processor_tests(test_name=None):
    """Run TestArXivPDFProcessor tests"""
    return run_test_suite(TestArXivPDFProcessor, test_name)

run_pdf_downloader_tests(test_name=None)

Run TestRobustPDFDownloader tests

Source code in toolboxv2/mods/TruthSeeker/tests.py

def run_pdf_downloader_tests(test_name=None):
    """Run TestRobustPDFDownloader tests"""
    return run_test_suite(TestRobustPDFDownloader, test_name)

run_specific_test(test_class, test_name)

Run a specific test from a test class

Source code in toolboxv2/mods/TruthSeeker/tests.py

def run_specific_test(test_class, test_name):
    """Run a specific test from a test class"""
    return run_test_suite(test_class, test_name)

run_test_suite(test_class=None, test_name=None, verbosity=2)

Run specific test class or test case.

Parameters:

Name	Type	Description	Default
test_class		The test class to run (optional)	None
test_name		Specific test method name to run (optional)	None
verbosity		Output detail level (default=2)	2

Returns:

Type	Description
	TestResult object

Source code in toolboxv2/mods/TruthSeeker/tests.py

def run_test_suite(test_class=None, test_name=None, verbosity=2):
    """
    Run specific test class or test case.

    Args:
        test_class: The test class to run (optional)
        test_name: Specific test method name to run (optional)
        verbosity: Output detail level (default=2)

    Returns:
        TestResult object
    """
    loader = unittest.TestLoader()
    suite = unittest.TestSuite()

    if test_class and test_name:
        # Run specific test method
        suite.addTest(test_class(test_name))
    elif test_class:
        # Run all tests in the class
        suite.addTests(loader.loadTestsFromTestCase(test_class))
    else:
        # Run all tests
        suite.addTests(loader.loadTestsFromModule(sys.modules[__name__]))

    runner = unittest.TextTestRunner(verbosity=verbosity)
    return runner.run(suite)

run_truth_seeker_tests(test_name=None)

Run TestTruthSeeker tests

Source code in toolboxv2/mods/TruthSeeker/tests.py

def run_truth_seeker_tests(test_name=None):
    """Run TestTruthSeeker tests"""
    return run_test_suite(TestTruthSeeker, test_name)

test_arxiv_search()

Run only ArXiv search tests

Source code in toolboxv2/mods/TruthSeeker/tests.py

@default_test
def test_arxiv_search():
    """Run only ArXiv search tests"""
    return run_specific_test(
        TestArXivPDFProcessor,
        'test_search_and_process_papers'
    )

test_pdf_download()

Run only PDF download tests

Source code in toolboxv2/mods/TruthSeeker/tests.py

@default_test
def test_pdf_download():
    """Run only PDF download tests"""
    return run_specific_test(
        TestRobustPDFDownloader,
        'test_download_pdf_success'
    )

test_truth_seeker()

Run only PDF download tests

Source code in toolboxv2/mods/TruthSeeker/tests.py

@default_test
def test_truth_seeker():
    """Run only PDF download tests"""
    return run_specific_test(
        TestTruthSeeker,
        'test_truth_seeker_success'
    )

UltimateTTT

UltimateTTTGameEngine

Source code in toolboxv2/mods/UltimateTTT.py

class UltimateTTTGameEngine:  # Renamed for clarity
    def __init__(self, game_state: GameState):
        self.gs = game_state
        self.size = game_state.config.grid_size

    def _check_line_for_win(self, line: list[CellState | BoardWinner],
                            symbol_to_check: CellState | BoardWinner) -> bool:
        if not line or line[0] == CellState.EMPTY or line[0] == BoardWinner.NONE:
            return False
        return all(cell == symbol_to_check for cell in line)

    def _get_board_winner_symbol(self, board: list[list[CellState | BoardWinner]],
                                 symbol_class: type[CellState] | type[BoardWinner]) -> CellState | BoardWinner | None:
        symbols_to_try = [symbol_class.X, symbol_class.O]
        for symbol in symbols_to_try:
            # Rows
            for r in range(self.size):
                if self._check_line_for_win([board[r][c] for c in range(self.size)], symbol): return symbol
            # Columns
            for c in range(self.size):
                if self._check_line_for_win([board[r][c] for r in range(self.size)], symbol): return symbol
            # Diagonals
            if self._check_line_for_win([board[i][i] for i in range(self.size)], symbol): return symbol
            if self._check_line_for_win([board[i][self.size - 1 - i] for i in range(self.size)], symbol): return symbol
        return None  # No winner

    def _is_board_full(self, board: list[list[CellState | BoardWinner]],
                       empty_value: CellState | BoardWinner) -> bool:
        return all(cell != empty_value for row in board for cell in row)

    def _determine_local_board_result(self, global_r: int, global_c: int) -> BoardWinner:
        if self.gs.global_board_winners[global_r][global_c] != BoardWinner.NONE:
            return self.gs.global_board_winners[global_r][global_c]

        local_board_cells = self.gs.local_boards_state[global_r][global_c]
        winner_symbol = self._get_board_winner_symbol(local_board_cells, CellState)
        if winner_symbol:
            return BoardWinner(winner_symbol.value)  # Convert CellState.X to BoardWinner.X
        if self._is_board_full(local_board_cells, CellState.EMPTY):
            return BoardWinner.DRAW
        return BoardWinner.NONE

    def _update_local_winner_and_check_global(self, global_r: int, global_c: int):
        new_local_winner = self._determine_local_board_result(global_r, global_c)
        if new_local_winner != BoardWinner.NONE and self.gs.global_board_winners[global_r][
            global_c] == BoardWinner.NONE:
            self.gs.global_board_winners[global_r][global_c] = new_local_winner
            self._check_for_overall_game_end()

    def _check_for_overall_game_end(self):
        if self.gs.status == GameStatus.FINISHED: return

        winner_board_symbol = self._get_board_winner_symbol(self.gs.global_board_winners, BoardWinner)
        if winner_board_symbol:  # This is BoardWinner.X or BoardWinner.O
            self.gs.overall_winner_symbol = PlayerSymbol(winner_board_symbol.value)  # Convert to PlayerSymbol
            self.gs.status = GameStatus.FINISHED
            return

        if self._is_board_full(self.gs.global_board_winners, BoardWinner.NONE):
            self.gs.is_draw = True
            self.gs.status = GameStatus.FINISHED

    def _determine_next_forced_board(self, last_move_local_r: int, last_move_local_c: int) -> tuple[int, int] | None:
        target_gr, target_gc = last_move_local_r, last_move_local_c

        if self.gs.global_board_winners[target_gr][target_gc] == BoardWinner.NONE and \
            not self._is_local_board_full(self.gs.local_boards_state[target_gr][target_gc], CellState.EMPTY):
            return (target_gr, target_gc)
        return None  # Play anywhere valid

    def _is_local_board_full(self, local_board_cells: list[list[CellState]], cell_type=CellState.EMPTY) -> bool:
        """Checks if a specific local board (passed as a 2D list of CellState) is full."""
        for r in range(self.size):
            for c in range(self.size):
                if local_board_cells[r][c] == cell_type:
                    return False
        return True

    def add_player(self, player_id: str, player_name: str,
                   is_npc: bool = False, npc_difficulty: NPCDifficulty | None = None) -> bool:
        if len(self.gs.players) >= 2:
            self.gs.last_error_message = "Game is already full (2 players max)."
            return False

        # Reconnect logic for existing player (human or NPC if that makes sense)
        existing_player = self.gs.get_player_info(player_id)
        if existing_player:
            if not existing_player.is_connected:
                existing_player.is_connected = True
                # If NPC "reconnects", ensure its properties are correct (though unlikely scenario for NPC)
                if is_npc:
                    existing_player.is_npc = True
                    existing_player.npc_difficulty = npc_difficulty
                    existing_player.name = player_name  # Update name if it changed for NPC

                self.gs.last_error_message = None
                self.gs.updated_at = datetime.now(UTC)

                if len(self.gs.players) == 2 and all(p.is_connected for p in self.gs.players) and \
                    self.gs.status == GameStatus.WAITING_FOR_OPPONENT:  # Should not be waiting if NPC is P2
                    self.gs.status = GameStatus.IN_PROGRESS
                    player_x_info = next(p for p in self.gs.players if p.symbol == PlayerSymbol.X)
                    self.gs.current_player_id = player_x_info.id
                    self.gs.waiting_since = None
                return True
            else:  # Player ID exists and is already connected
                self.gs.last_error_message = f"Player with ID {player_id} is already in the game and connected."
                return False

        # Adding a new player
        symbol = PlayerSymbol.X if not self.gs.players else PlayerSymbol.O

        # Construct PlayerInfo with NPC details if applicable
        player_info_data = {
            "id": player_id,
            "symbol": symbol,
            "name": player_name,
            "is_connected": True,  # NPCs are always "connected"
            "is_npc": is_npc
        }
        if is_npc and npc_difficulty:
            player_info_data["npc_difficulty"] = npc_difficulty

        new_player = PlayerInfo(**player_info_data)
        self.gs.players.append(new_player)
        self.gs.last_error_message = None

        if len(self.gs.players) == 1:  # First player added
            if self.gs.mode == GameMode.ONLINE:
                self.gs.status = GameStatus.WAITING_FOR_OPPONENT
                self.gs.current_player_id = player_id
                self.gs.waiting_since = datetime.now(UTC)
            # For local mode with P1, we wait for P2 (human or NPC) to be added
            # No status change yet, current_player_id not set until P2 joins

        elif len(self.gs.players) == 2:  # Both players now present
            self.gs.status = GameStatus.IN_PROGRESS
            player_x_info = next(p for p in self.gs.players if p.symbol == PlayerSymbol.X)
            self.gs.current_player_id = player_x_info.id  # X always starts
            self.gs.next_forced_global_board = None
            self.gs.waiting_since = None

            # If the second player added is an NPC and it's their turn (e.g. P1 is human, P2 is NPC, P1 made a move)
            # This specific logic is more for when make_move hands over to an NPC.
            # Here, we just set up the game. X (P1) will make the first move.

        self.gs.updated_at = datetime.now(UTC)
        return True

    def make_move(self, move: Move) -> bool:
        self.gs.last_error_message = None

        if self.gs.status != GameStatus.IN_PROGRESS:
            self.gs.last_error_message = "Game is not in progress."
            return False

        player_info = self.gs.get_player_info(move.player_id)
        if not player_info or move.player_id != self.gs.current_player_id:
            self.gs.last_error_message = "Not your turn or invalid player."
            return False

        s = self.size
        if not (0 <= move.global_row < s and 0 <= move.global_col < s and \
                0 <= move.local_row < s and 0 <= move.local_col < s):
            self.gs.last_error_message = f"Coordinates out of bounds for {s}x{s} grid."
            return False

        gr, gc, lr, lc = move.global_row, move.global_col, move.local_row, move.local_col

        if self.gs.next_forced_global_board and (gr, gc) != self.gs.next_forced_global_board:
            self.gs.last_error_message = f"Must play in global board {self.gs.next_forced_global_board}."
            return False

        if self.gs.global_board_winners[gr][gc] != BoardWinner.NONE:
            self.gs.last_error_message = f"Local board ({gr},{gc}) is already decided."
            return False
        if self.gs.local_boards_state[gr][gc][lr][lc] != CellState.EMPTY:
            self.gs.last_error_message = f"Cell ({gr},{gc})-({lr},{lc}) is already empty."  # Should be 'not empty' or 'occupied'
            # Correction:
            self.gs.last_error_message = f"Cell ({gr},{gc})-({lr},{lc}) is already occupied."
            return False

        self.gs.local_boards_state[gr][gc][lr][lc] = CellState(player_info.symbol.value)
        self.gs.moves_history.append(move)

        self._update_local_winner_and_check_global(gr, gc)

        if self.gs.status == GameStatus.FINISHED:
            self.gs.next_forced_global_board = None
        else:
            opponent_info = self.gs.get_opponent_info(self.gs.current_player_id)
            self.gs.current_player_id = opponent_info.id
            self.gs.next_forced_global_board = self._determine_next_forced_board(lr, lc)

            if self.gs.next_forced_global_board is None:
                is_any_move_possible = any(
                    self.gs.global_board_winners[r_idx][c_idx] == BoardWinner.NONE and \
                    not self._is_local_board_full(self.gs.local_boards_state[r_idx][c_idx], CellState.EMPTY)
                    for r_idx in range(s) for c_idx in range(s)
                )
                if not is_any_move_possible:
                    self._check_for_overall_game_end()
                    if self.gs.status != GameStatus.FINISHED:
                        self.gs.is_draw = True
                        self.gs.status = GameStatus.FINISHED

        self.gs.updated_at = datetime.now(UTC)
        self.gs.last_made_move_coords = (move.global_row, move.global_col, move.local_row, move.local_col)

        return True

    def handle_player_disconnect(self, player_id: str):
        player = self.gs.get_player_info(player_id)
        app = get_app(GAME_NAME)  # Hol dir die App-Instanz
        if player:
            if not player.is_connected:  # Already marked as disconnected
                app.logger.info(f"Player {player_id} was already marked as disconnected from game {self.gs.game_id}.")
                return

            player.is_connected = False
            self.gs.updated_at = datetime.now(UTC)
            app.logger.info(f"Player {player_id} disconnected from game {self.gs.game_id}. Name: {player.name}")

            if self.gs.mode == GameMode.ONLINE:
                if self.gs.status == GameStatus.IN_PROGRESS:
                    opponent = self.gs.get_opponent_info(player_id)
                    if opponent and opponent.is_connected:
                        self.gs.status = GameStatus.ABORTED  # Use ABORTED as "paused"
                        self.gs.player_who_paused = player_id  # Store who disconnected
                        # This message is for the game state, will be seen by the other player via SSE
                        self.gs.last_error_message = f"Player {player.name} disconnected. Waiting for them to rejoin."
                        app.logger.info(
                            f"Game {self.gs.game_id} PAUSED, waiting for {player.name} ({player_id}) to reconnect.")
                    else:
                        # Opponent also disconnected or was already gone
                        self.gs.status = GameStatus.ABORTED
                        self.gs.last_error_message = "Both players disconnected. Game aborted."
                        self.gs.player_who_paused = None  # No specific player to wait for
                        app.logger.info(
                            f"Game {self.gs.game_id} ABORTED, both players (or last active player) disconnected.")
                elif self.gs.status == GameStatus.WAITING_FOR_OPPONENT:
                    # If the creator (P1) disconnects while waiting for P2
                    if len(self.gs.players) == 1 and self.gs.players[0].id == player_id:
                        self.gs.status = GameStatus.ABORTED
                        self.gs.last_error_message = "Game creator disconnected before opponent joined. Game aborted."
                        self.gs.player_who_paused = None
                        app.logger.info(
                            f"Game {self.gs.game_id} ABORTED, creator {player.name} ({player_id}) disconnected while WAITING_FOR_OPPONENT.")
                elif self.gs.status == GameStatus.ABORTED and self.gs.player_who_paused:
                    # Game was already paused (e.g. P1 disconnected), and now P2 (the waiting one) disconnects
                    if self.gs.player_who_paused != player_id:  # Ensure it's the other player
                        self.gs.last_error_message = "Other player also disconnected during pause. Game aborted."
                        self.gs.player_who_paused = None  # No one specific to wait for now
                        app.logger.info(
                            f"Game {self.gs.game_id} ABORTED, waiting player {player.name} ({player_id}) disconnected.")

    def handle_player_reconnect(self, player_id: str) -> bool:
        player = self.gs.get_player_info(player_id)
        app = get_app(GAME_NAME)
        if not player:
            app.logger.warning(f"Reconnect attempt for unknown player {player_id} in game {self.gs.game_id}.")
            return False

        if player.is_connected:
            app.logger.info(
                f"Player {player.name} ({player_id}) attempted reconnect but was already marked as connected to game {self.gs.game_id}.")
            if self.gs.status == GameStatus.ABORTED and self.gs.player_who_paused == player_id:
                opponent = self.gs.get_opponent_info(player_id)
                if opponent and opponent.is_connected:
                    self.gs.status = GameStatus.IN_PROGRESS
                    self.gs.last_error_message = f"Connection for {player.name} re-established. Game resumed."
                    self.gs.player_who_paused = None
                    self.gs.updated_at = datetime.now(UTC)
                    app.logger.info(
                        f"Game {self.gs.game_id} resumed as already-connected pauser {player.name} re-interacted.")
                else:
                    self.gs.last_error_message = f"Welcome back, {player.name}! Your opponent is still not connected."
            return True

        player.is_connected = True
        self.gs.updated_at = datetime.now(UTC)
        app.logger.info(
            f"Player {player.name} ({player_id}) reconnected to game {self.gs.game_id}. Previous status: {self.gs.status}, Paused by: {self.gs.player_who_paused}")

        if self.gs.status == GameStatus.ABORTED:
            if self.gs.player_who_paused == player_id:  # The player who caused the pause has reconnected
                opponent = self.gs.get_opponent_info(player_id)
                if opponent and opponent.is_connected:
                    self.gs.status = GameStatus.IN_PROGRESS
                    self.gs.last_error_message = f"Player {player.name} reconnected. Game resumed!"
                    self.gs.player_who_paused = None
                    app.logger.info(
                        f"Game {self.gs.game_id} RESUMED. Pauser {player.name} reconnected, opponent {opponent.name} is present.")
                else:  # Pauser reconnected, opponent (still) gone or never joined (if P1 disconnected from WAITING)
                    if not opponent and len(
                        self.gs.players) == 1:  # P1 reconnected to a game they created but no P2 yet
                        self.gs.status = GameStatus.WAITING_FOR_OPPONENT
                        self.gs.player_who_paused = None
                        self.gs.current_player_id = player_id
                        self.gs.last_error_message = f"Creator {player.name} reconnected. Waiting for opponent."
                        self.gs.waiting_since = datetime.now(UTC)  # Reset waiting timer
                    elif opponent:  # Opponent was there but is now disconnected
                        self.gs.player_who_paused = opponent.id  # Now waiting for the other person
                        self.gs.last_error_message = f"Welcome back, {player.name}! Your opponent ({opponent.name}) is not connected. Game remains paused."
                        app.logger.info(
                            f"Game {self.gs.game_id} still PAUSED. {player.name} reconnected, but opponent {opponent.name} is NOT. Waiting for {opponent.name}.")
                    else:  # Should be rare: 2 players in list, but opponent object not found for P1
                        self.gs.last_error_message = f"Welcome back, {player.name}! Opponent details unclear. Game remains paused."


            elif self.gs.player_who_paused and self.gs.player_who_paused != player_id:
                # The *other* player reconnected, while game was paused for initial pauser.
                initial_pauser_info = self.gs.get_player_info(self.gs.player_who_paused)
                if initial_pauser_info and initial_pauser_info.is_connected:  # This implies both are now connected.
                    self.gs.status = GameStatus.IN_PROGRESS
                    self.gs.last_error_message = "Both players are now connected. Game resumed!"
                    self.gs.player_who_paused = None
                    app.logger.info(
                        f"Game {self.gs.game_id} RESUMED. Waiting player {player.name} reconnected, initial pauser {initial_pauser_info.name} also present.")
                else:
                    self.gs.last_error_message = f"Welcome back, {player.name}! Still waiting for {initial_pauser_info.name if initial_pauser_info else 'the other player'} to reconnect."
                    app.logger.info(
                        f"Game {self.gs.game_id} still PAUSED. Player {player.name} reconnected, but still waiting for original pauser {self.gs.player_who_paused}.")

            else:  # game is ABORTED but no specific player_who_paused (hard abort by timeout or both disconnected)
                if len(self.gs.players) == 2:  # Was a two-player game
                    opponent = self.gs.get_opponent_info(player_id)
                    if opponent:
                        # Revive the game to a paused state, waiting for the other player
                        self.gs.player_who_paused = opponent.id
                        self.gs.status = GameStatus.ABORTED  # Remains aborted, but now specifically for opponent
                        self.gs.last_error_message = f"Welcome back, {player.name}! Game was fully aborted. Now waiting for {opponent.name} to rejoin."
                        app.logger.info(
                            f"Game {self.gs.game_id} REVIVED from HARD ABORT by {player.name}. Now paused, waiting for {opponent.name} ({opponent.id}).")
                    else:  # Should not happen if two players were in game and player_id is one of them
                        self.gs.last_error_message = f"Player {player.name} reconnected, but game state is inconsistent (opponent not found)."
                        app.logger.warning(
                            f"Game {self.gs.game_id} HARD ABORT revival by {player.name} failed, opponent info missing.")
                elif len(self.gs.players) == 1 and self.gs.players[0].id == player_id:
                    # P1 created, P1 disconnected, game WAITING_FOR_OPPONENT timed out & hard aborted. P1 tries to rejoin.
                    self.gs.status = GameStatus.WAITING_FOR_OPPONENT
                    self.gs.player_who_paused = None
                    self.gs.current_player_id = player_id
                    self.gs.last_error_message = f"Creator {player.name} reconnected. Waiting for opponent."
                    self.gs.waiting_since = datetime.now(UTC)  # Reset waiting timer
                    app.logger.info(
                        f"Game {self.gs.game_id} (previously hard aborted while waiting) revived by creator {player.name}. Now WAITING_FOR_OPPONENT.")
                else:
                    self.gs.last_error_message = f"Player {player.name} reconnected, but the game was aborted and cannot be revived in its current player configuration."
                    app.logger.info(
                        f"Game {self.gs.game_id} HARD ABORTED. Player {player.name} reconnected, but game cannot resume in current configuration.")


        elif self.gs.status == GameStatus.IN_PROGRESS:
            opponent = self.gs.get_opponent_info(player_id)
            if not opponent or not opponent.is_connected:
                self.gs.status = GameStatus.ABORTED
                self.gs.player_who_paused = opponent.id if opponent else None
                self.gs.last_error_message = f"Welcome back, {player.name}! Your opponent disconnected while you were away. Waiting for them."
                app.logger.info(
                    f"Game {self.gs.game_id} transitions to PAUSED. {player.name} reconnected to IN_PROGRESS, but opponent {opponent.id if opponent else 'N/A'} is gone.")
            else:
                self.gs.last_error_message = f"Player {player.name} re-established connection during active game."
                app.logger.info(
                    f"Player {player.name} ({player_id}) re-established connection to IN_PROGRESS game {self.gs.game_id}.")

        elif self.gs.status == GameStatus.WAITING_FOR_OPPONENT:
            if len(self.gs.players) == 1 and self.gs.players[0].id == player_id:
                self.gs.last_error_message = f"Creator {player.name} reconnected. Still waiting for opponent."
                self.gs.current_player_id = player_id
                self.gs.waiting_since = datetime.now(UTC)  # Reset waiting timer
                app.logger.info(
                    f"Creator {player.name} ({player_id}) reconnected to WAITING_FOR_OPPONENT game {self.gs.game_id}.")
            else:
                app.logger.warning(
                    f"Non-creator {player.name} or unexpected player count for reconnect to WAITING_FOR_OPPONENT game {self.gs.game_id}.")

        return True

WebSocketManager

Tools

Bases: MainTool

Production-ready WebSocketManager Tool.

Source code in toolboxv2/mods/WebSocketManager.py

class Tools(MainTool):
    """Production-ready WebSocketManager Tool."""

    def __init__(self, app=None):
        self.version = "2.0.0"
        self.name = "WebSocketManager"
        self.color = "BLUE"

        if app is None:
            app = get_app()
        self.logger = app.logger if app else logging.getLogger(self.name)

        # Core components
        self.server: Optional[WebSocketServer] = None
        self.clients: Dict[str, WebSocketClient] = {}
        self.pools: Dict[str, WebSocketPool] = {}

        # Tools interface
        self.tools = {
            "all": [
                ["version", "Show version"],
                ["create_server", "Create WebSocket server"],
                ["create_client", "Create WebSocket client"],
                ["create_pool", "Create connection pool"],
                ["list_pools", "List all pools"],
                ["get_stats", "Get connection statistics"],
                ["health_check", "Perform health check"]
            ],
            "name": self.name,
            "version": self.show_version,
            #"create_server": self.create_server,
            "create_client": self.create_client,
            "create_pool": self.create_pool,
            "list_pools": self.list_pools,
            "get_stats": self.get_statistics,
            "health_check": self.health_check
        }

        MainTool.__init__(self, load=self.on_start, v=self.version,
                          tool=self.tools, name=self.name,
                          logs=self.logger, color=self.color,
                          on_exit=self.on_exit)

    def on_start(self):
        """Initialize the WebSocketManager."""
        self.logger.info("🚀 WebSocketManager started")

    async def on_exit(self):
        """Cleanup on exit."""
        self.logger.info("🔄 Shutting down WebSocketManager")

        # Stop server
        if self.server:
            await self.server.stop()

        # Disconnect all clients
        for client in self.clients.values():
            await client.disconnect()

        self.logger.info("✅ WebSocketManager shutdown complete")

    def show_version(self):
        """Show current version."""
        return self.version

    async def create_server(self, host: str = "localhost", port: int = 8765,
                            non_blocking: bool = False) -> WebSocketServer:
        """Create and start a WebSocket server."""
        if non_blocking is None:
            return
        if 'test' in host:
            return
        if self.server is None:
            self.server = WebSocketServer(host, port)
            await self.server.start(non_blocking)
        return self.server

    def create_client(self, client_id: str) -> WebSocketClient:
        """Create a WebSocket client."""
        if client_id not in self.clients:
            self.clients[client_id] = WebSocketClient(client_id, self.logger)
        return self.clients[client_id]

    def create_pool(self, pool_id: str) -> WebSocketPool:
        """Create a standalone connection pool."""
        if pool_id not in self.pools:
            self.pools[pool_id] = WebSocketPool(pool_id)
        return self.pools[pool_id]

    def list_pools(self) -> Dict[str, Dict[str, Any]]:
        """List all connection pools with stats."""
        pools_info = {}

        # Server pools
        if self.server:
            for pool_id, pool in self.server.pools.items():
                pools_info[f"server.{pool_id}"] = {
                    "type": "server_pool",
                    "connections": pool.get_connection_count(),
                    "connection_ids": pool.get_connection_ids()
                }

        # Standalone pools
        for pool_id, pool in self.pools.items():
            pools_info[pool_id] = {
                "type": "standalone_pool",
                "connections": pool.get_connection_count(),
                "connection_ids": pool.get_connection_ids()
            }

        return pools_info

    def get_statistics(self) -> Dict[str, Any]:
        """Get comprehensive statistics."""
        stats = {
            "server": {
                "running": self.server is not None,
                "pools": len(self.server.pools) if self.server else 0,
                "total_connections": sum(
                    pool.get_connection_count()
                    for pool in (self.server.pools.values() if self.server else [])
                )
            },
            "clients": {
                "total": len(self.clients),
                "connected": sum(
                    1 for client in self.clients.values()
                    if client.state == ConnectionState.CONNECTED
                ),
                "states": {
                    state.value: sum(
                        1 for client in self.clients.values()
                        if client.state == state
                    ) for state in ConnectionState
                }
            },
            "pools": {
                "standalone": len(self.pools),
                "total_connections": sum(
                    pool.get_connection_count()
                    for pool in self.pools.values()
                )
            }
        }
        return stats

    async def health_check(self) -> Dict[str, Any]:
        """Perform comprehensive health check."""
        health = {
            "overall": "healthy",
            "server": "not_running" if not self.server else "running",
            "clients": {},
            "issues": []
        }

        # Check clients
        for client_id, client in self.clients.items():
            if client.state == ConnectionState.CONNECTED:
                # Perform actual health check if possible
                try:
                    if client.ws and not client.ws.closed:
                        health["clients"][client_id] = "healthy"
                    else:
                        health["clients"][client_id] = "unhealthy"
                        health["issues"].append(f"Client {client_id} connection closed")
                except Exception as e:
                    health["clients"][client_id] = "error"
                    health["issues"].append(f"Client {client_id}: {str(e)}")
            else:
                health["clients"][client_id] = client.state.value

        if health["issues"]:
            health["overall"] = "degraded"

        return health

    # Utility methods for easy access
    def get_server_pool(self, pool_id: str) -> Optional[WebSocketPool]:
        """Get a server pool by ID."""
        return self.server.get_pool(pool_id) if self.server else None

    def get_client(self, client_id: str) -> Optional[WebSocketClient]:
        """Get a client by ID."""
        return self.clients.get(client_id)

    async def broadcast_to_pool(self, pool_id: str, event: str, data: Dict[str, Any]) -> int:
        """Broadcast message to all connections in a pool."""
        message = WebSocketMessage(event=event, data=data).to_json()

        # Try server pool first
        if self.server:
            pool = self.server.get_pool(pool_id)
            if pool:
                return await pool.broadcast(message)

        # Try standalone pool
        pool = self.pools.get(pool_id)
        if pool:
            return await pool.broadcast(message)

        return 0

broadcast_to_pool(pool_id, event, data) async

Broadcast message to all connections in a pool.

Source code in toolboxv2/mods/WebSocketManager.py

async def broadcast_to_pool(self, pool_id: str, event: str, data: Dict[str, Any]) -> int:
    """Broadcast message to all connections in a pool."""
    message = WebSocketMessage(event=event, data=data).to_json()

    # Try server pool first
    if self.server:
        pool = self.server.get_pool(pool_id)
        if pool:
            return await pool.broadcast(message)

    # Try standalone pool
    pool = self.pools.get(pool_id)
    if pool:
        return await pool.broadcast(message)

    return 0

create_client(client_id)

Create a WebSocket client.

Source code in toolboxv2/mods/WebSocketManager.py

def create_client(self, client_id: str) -> WebSocketClient:
    """Create a WebSocket client."""
    if client_id not in self.clients:
        self.clients[client_id] = WebSocketClient(client_id, self.logger)
    return self.clients[client_id]

create_pool(pool_id)

Create a standalone connection pool.

Source code in toolboxv2/mods/WebSocketManager.py

def create_pool(self, pool_id: str) -> WebSocketPool:
    """Create a standalone connection pool."""
    if pool_id not in self.pools:
        self.pools[pool_id] = WebSocketPool(pool_id)
    return self.pools[pool_id]

create_server(host='localhost', port=8765, non_blocking=False) async

Create and start a WebSocket server.

Source code in toolboxv2/mods/WebSocketManager.py

async def create_server(self, host: str = "localhost", port: int = 8765,
                        non_blocking: bool = False) -> WebSocketServer:
    """Create and start a WebSocket server."""
    if non_blocking is None:
        return
    if 'test' in host:
        return
    if self.server is None:
        self.server = WebSocketServer(host, port)
        await self.server.start(non_blocking)
    return self.server

get_client(client_id)

Get a client by ID.

Source code in toolboxv2/mods/WebSocketManager.py

def get_client(self, client_id: str) -> Optional[WebSocketClient]:
    """Get a client by ID."""
    return self.clients.get(client_id)

get_server_pool(pool_id)

Get a server pool by ID.

Source code in toolboxv2/mods/WebSocketManager.py

def get_server_pool(self, pool_id: str) -> Optional[WebSocketPool]:
    """Get a server pool by ID."""
    return self.server.get_pool(pool_id) if self.server else None

get_statistics()

Get comprehensive statistics.

Source code in toolboxv2/mods/WebSocketManager.py

def get_statistics(self) -> Dict[str, Any]:
    """Get comprehensive statistics."""
    stats = {
        "server": {
            "running": self.server is not None,
            "pools": len(self.server.pools) if self.server else 0,
            "total_connections": sum(
                pool.get_connection_count()
                for pool in (self.server.pools.values() if self.server else [])
            )
        },
        "clients": {
            "total": len(self.clients),
            "connected": sum(
                1 for client in self.clients.values()
                if client.state == ConnectionState.CONNECTED
            ),
            "states": {
                state.value: sum(
                    1 for client in self.clients.values()
                    if client.state == state
                ) for state in ConnectionState
            }
        },
        "pools": {
            "standalone": len(self.pools),
            "total_connections": sum(
                pool.get_connection_count()
                for pool in self.pools.values()
            )
        }
    }
    return stats

health_check() async

Perform comprehensive health check.

Source code in toolboxv2/mods/WebSocketManager.py

async def health_check(self) -> Dict[str, Any]:
    """Perform comprehensive health check."""
    health = {
        "overall": "healthy",
        "server": "not_running" if not self.server else "running",
        "clients": {},
        "issues": []
    }

    # Check clients
    for client_id, client in self.clients.items():
        if client.state == ConnectionState.CONNECTED:
            # Perform actual health check if possible
            try:
                if client.ws and not client.ws.closed:
                    health["clients"][client_id] = "healthy"
                else:
                    health["clients"][client_id] = "unhealthy"
                    health["issues"].append(f"Client {client_id} connection closed")
            except Exception as e:
                health["clients"][client_id] = "error"
                health["issues"].append(f"Client {client_id}: {str(e)}")
        else:
            health["clients"][client_id] = client.state.value

    if health["issues"]:
        health["overall"] = "degraded"

    return health

list_pools()

List all connection pools with stats.

Source code in toolboxv2/mods/WebSocketManager.py

def list_pools(self) -> Dict[str, Dict[str, Any]]:
    """List all connection pools with stats."""
    pools_info = {}

    # Server pools
    if self.server:
        for pool_id, pool in self.server.pools.items():
            pools_info[f"server.{pool_id}"] = {
                "type": "server_pool",
                "connections": pool.get_connection_count(),
                "connection_ids": pool.get_connection_ids()
            }

    # Standalone pools
    for pool_id, pool in self.pools.items():
        pools_info[pool_id] = {
            "type": "standalone_pool",
            "connections": pool.get_connection_count(),
            "connection_ids": pool.get_connection_ids()
        }

    return pools_info

on_exit() async

Cleanup on exit.

Source code in toolboxv2/mods/WebSocketManager.py

async def on_exit(self):
    """Cleanup on exit."""
    self.logger.info("🔄 Shutting down WebSocketManager")

    # Stop server
    if self.server:
        await self.server.stop()

    # Disconnect all clients
    for client in self.clients.values():
        await client.disconnect()

    self.logger.info("✅ WebSocketManager shutdown complete")

on_start()

Initialize the WebSocketManager.

Source code in toolboxv2/mods/WebSocketManager.py

def on_start(self):
    """Initialize the WebSocketManager."""
    self.logger.info("🚀 WebSocketManager started")

show_version()

Show current version.

Source code in toolboxv2/mods/WebSocketManager.py

def show_version(self):
    """Show current version."""
    return self.version

WebSocketClient

Robust WebSocket client with automatic reconnection.

Source code in toolboxv2/mods/WebSocketManager.py

class WebSocketClient:
    """Robust WebSocket client with automatic reconnection."""

    def __init__(self, client_id: str, logger: Optional[logging.Logger] = None):
        self.client_id = client_id
        self.logger = logger or logging.getLogger(f"WSClient.{client_id}")

        # Connection management
        self.ws: Optional[Any] = None
        self.server_url: Optional[str] = None
        self.state = ConnectionState.DISCONNECTED

        # Tasks and control
        self.should_reconnect = True
        self.reconnect_attempts = 0
        self.max_reconnect_attempts = 10
        self.connection_task: Optional[asyncio.Task] = None
        self.ping_task: Optional[asyncio.Task] = None

        # Message handling
        self.message_handlers: Dict[str, Callable] = {}
        self.message_queue = asyncio.Queue()

    async def connect(self, server_url: str, timeout: float = 30.0) -> bool:
        """Connect to WebSocket server."""
        if self.state in [ConnectionState.CONNECTED, ConnectionState.CONNECTING]:
            return True

        self.server_url = server_url
        self.state = ConnectionState.CONNECTING
        self.should_reconnect = True

        try:
            self.logger.info(f"Connecting to {server_url}")
            self.ws = await asyncio.wait_for(ws_connect(server_url), timeout=timeout)

            self.state = ConnectionState.CONNECTED
            self.reconnect_attempts = 0

            # Start background tasks
            self.connection_task = asyncio.create_task(self._listen_loop())
            self.ping_task = asyncio.create_task(self._ping_loop())

            self.logger.info("✅ Connected successfully")
            return True

        except Exception as e:
            self.logger.error(f"❌ Connection failed: {e}")
            self.state = ConnectionState.DISCONNECTED
            return False

    async def disconnect(self) -> None:
        """Gracefully disconnect."""
        self.should_reconnect = False
        self.state = ConnectionState.CLOSED

        # Cancel tasks
        for task in [self.connection_task, self.ping_task]:
            if task and not task.done():
                task.cancel()

        # Close connection
        if self.ws:
            try:
                await self.ws.close()
            except Exception:
                pass
            self.ws = None

        self.logger.info("✅ Disconnected")

    def register_handler(self, event: str, handler: Callable[[WebSocketMessage], Awaitable[None]]) -> None:
        """Register a message handler for specific events."""
        self.message_handlers[event] = handler
        self.logger.info(f"Registered handler for event: {event}")

    async def send_message(self, event: str, data: Dict[str, Any]) -> bool:
        """Send a message to the server."""
        if self.state != ConnectionState.CONNECTED or not self.ws:
            self.logger.warning("Cannot send message: not connected")
            return False

        try:
            message = WebSocketMessage(event=event, data=data)
            await self.ws.send(message.to_json())
            return True
        except Exception as e:
            self.logger.error(f"Failed to send message: {e}")
            await self._trigger_reconnect()
            return False

    async def _listen_loop(self) -> None:
        """Main message listening loop."""
        while self.should_reconnect and self.ws:
            try:
                # Kürzere Timeouts für bessere Responsivität
                message_raw = await asyncio.wait_for(self.ws.recv(), timeout=1.0)

                # Handle message in background task to prevent blocking
                asyncio.create_task(self._handle_message(message_raw))

            except asyncio.TimeoutError:
                # Check connection health during timeout
                if self.ws and self.ws.closed:
                    self.logger.warning("Connection closed during timeout")
                    break
                continue
            except ConnectionClosed:
                self.logger.warning("Connection closed by server")
                break
            except Exception as e:
                self.logger.error(f"Listen loop error: {e}")
                break

        if self.should_reconnect:
            await self._trigger_reconnect()

    async def _handle_message(self, message_raw: str) -> None:
        """Handle incoming messages."""
        try:
            message = WebSocketMessage.from_json(message_raw)

            if message.event in self.message_handlers:
                await self.message_handlers[message.event](message)
            else:
                self.logger.debug(f"No handler for event: {message.event}")

        except Exception as e:
            self.logger.error(f"Message handling error: {e}")

    async def _ping_loop(self) -> None:
        """Periodic ping to maintain connection."""
        while self.should_reconnect and self.state == ConnectionState.CONNECTED:
            try:
                await asyncio.sleep(20)  # Ping every 20 seconds

                if self.ws and not self.ws.closed:
                    pong_waiter = await self.ws.ping()
                    await asyncio.wait_for(pong_waiter, timeout=10.0)
                    self.logger.debug("📡 Ping successful")
                else:
                    break

            except asyncio.TimeoutError:
                self.logger.error("Ping timeout - connection may be dead")
                break
            except Exception as e:
                self.logger.error(f"Ping failed: {e}")
                break

        if self.should_reconnect:
            await self._trigger_reconnect()

    async def _trigger_reconnect(self) -> None:
        """Trigger reconnection with exponential backoff."""
        if self.state == ConnectionState.RECONNECTING:
            return

        self.state = ConnectionState.RECONNECTING
        self.logger.info("🔄 Starting reconnection...")

        while (self.should_reconnect and
               self.reconnect_attempts < self.max_reconnect_attempts):

            self.reconnect_attempts += 1
            delay = min(2 ** self.reconnect_attempts, 60)  # Max 60s delay

            self.logger.info(f"Reconnect attempt {self.reconnect_attempts} in {delay}s")
            await asyncio.sleep(delay)

            try:
                if await self.connect(self.server_url):
                    return
            except Exception as e:
                self.logger.error(f"Reconnect attempt failed: {e}")

        self.logger.error("❌ Max reconnection attempts reached")
        self.should_reconnect = False
        self.state = ConnectionState.DISCONNECTED

connect(server_url, timeout=30.0) async

Connect to WebSocket server.

Source code in toolboxv2/mods/WebSocketManager.py

async def connect(self, server_url: str, timeout: float = 30.0) -> bool:
    """Connect to WebSocket server."""
    if self.state in [ConnectionState.CONNECTED, ConnectionState.CONNECTING]:
        return True

    self.server_url = server_url
    self.state = ConnectionState.CONNECTING
    self.should_reconnect = True

    try:
        self.logger.info(f"Connecting to {server_url}")
        self.ws = await asyncio.wait_for(ws_connect(server_url), timeout=timeout)

        self.state = ConnectionState.CONNECTED
        self.reconnect_attempts = 0

        # Start background tasks
        self.connection_task = asyncio.create_task(self._listen_loop())
        self.ping_task = asyncio.create_task(self._ping_loop())

        self.logger.info("✅ Connected successfully")
        return True

    except Exception as e:
        self.logger.error(f"❌ Connection failed: {e}")
        self.state = ConnectionState.DISCONNECTED
        return False

disconnect() async

Gracefully disconnect.

Source code in toolboxv2/mods/WebSocketManager.py

async def disconnect(self) -> None:
    """Gracefully disconnect."""
    self.should_reconnect = False
    self.state = ConnectionState.CLOSED

    # Cancel tasks
    for task in [self.connection_task, self.ping_task]:
        if task and not task.done():
            task.cancel()

    # Close connection
    if self.ws:
        try:
            await self.ws.close()
        except Exception:
            pass
        self.ws = None

    self.logger.info("✅ Disconnected")

register_handler(event, handler)

Register a message handler for specific events.

Source code in toolboxv2/mods/WebSocketManager.py

def register_handler(self, event: str, handler: Callable[[WebSocketMessage], Awaitable[None]]) -> None:
    """Register a message handler for specific events."""
    self.message_handlers[event] = handler
    self.logger.info(f"Registered handler for event: {event}")

send_message(event, data) async

Send a message to the server.

Source code in toolboxv2/mods/WebSocketManager.py

async def send_message(self, event: str, data: Dict[str, Any]) -> bool:
    """Send a message to the server."""
    if self.state != ConnectionState.CONNECTED or not self.ws:
        self.logger.warning("Cannot send message: not connected")
        return False

    try:
        message = WebSocketMessage(event=event, data=data)
        await self.ws.send(message.to_json())
        return True
    except Exception as e:
        self.logger.error(f"Failed to send message: {e}")
        await self._trigger_reconnect()
        return False

WebSocketPool

Manages a pool of WebSocket connections with actions and message routing.

Source code in toolboxv2/mods/WebSocketManager.py

class WebSocketPool:
    """Manages a pool of WebSocket connections with actions and message routing."""

    def __init__(self, pool_id: str):
        self.pool_id = pool_id
        self.connections: Dict[str, Any] = {}
        self.actions: Dict[str, Callable] = {}
        self.global_actions: Dict[str, Callable] = {}
        self.metadata: Dict[str, Any] = {}
        self.logger = logging.getLogger(f"WSPool.{pool_id}")

    async def add_connection(self, connection_id: str, websocket: Any) -> None:
        """Add a WebSocket connection to the pool."""
        self.connections[connection_id] = websocket
        self.logger.info(f"Added connection {connection_id} (total: {len(self.connections)})")

    async def remove_connection(self, connection_id: str) -> None:
        """Remove a WebSocket connection from the pool."""
        if connection_id in self.connections:
            del self.connections[connection_id]
            self.logger.info(f"Removed connection {connection_id} (remaining: {len(self.connections)})")

    def register_action(self, action_name: str, handler: Callable,
                        connection_ids: Optional[List[str]] = None) -> None:
        """Register an action handler for specific connections or globally."""
        if connection_ids is None:
            self.global_actions[action_name] = handler
            self.logger.info(f"Registered global action: {action_name}")
        else:
            for conn_id in connection_ids:
                if conn_id not in self.actions:
                    self.actions[conn_id] = {}
                self.actions[conn_id][action_name] = handler
            self.logger.info(f"Registered action {action_name} for connections: {connection_ids}")

    async def handle_message(self, connection_id: str, message: str) -> None:
        """Route incoming messages to appropriate handlers."""
        try:
            ws_message = WebSocketMessage.from_json(message)
            action = ws_message.event

            # Handle ping/pong
            if action == 'ping':
                pong_message = WebSocketMessage(event='pong', data={})
                await self.send_to_connection(connection_id, pong_message.to_json())
                return

            # Try global actions first
            if action in self.global_actions:
                # Run in executor to prevent blocking
                loop = asyncio.get_event_loop()
                await loop.run_in_executor(
                    None,
                    lambda: asyncio.create_task(
                        self.global_actions[action](self.pool_id, connection_id, ws_message)
                    )
                )
            # Then try connection-specific actions
            elif connection_id in self.actions and action in self.actions[connection_id]:
                loop = asyncio.get_event_loop()
                await loop.run_in_executor(
                    None,
                    lambda: asyncio.create_task(
                        self.actions[connection_id][action](self.pool_id, connection_id, ws_message)
                    )
                )
            else:
                self.logger.warning(f"No handler for action '{action}' from {connection_id}")

        except json.JSONDecodeError:
            self.logger.error(f"Invalid JSON from {connection_id}: {message[:100]}")
        except Exception as e:
            self.logger.error(f"Error handling message from {connection_id}: {e}")

    async def broadcast(self, message: str, exclude_connection: Optional[str] = None) -> int:
        """Broadcast message to all connections in the pool."""
        sent_count = 0
        for conn_id, websocket in list(self.connections.items()):
            if conn_id != exclude_connection:
                try:
                    await websocket.send(message)
                    sent_count += 1
                except Exception as e:
                    self.logger.error(f"Failed to send to {conn_id}: {e}")
                    await self.remove_connection(conn_id)
        return sent_count

    async def send_to_connection(self, connection_id: str, message: str) -> bool:
        """Send message to a specific connection."""
        if connection_id in self.connections:
            try:
                await self.connections[connection_id].send(message)
                return True
            except Exception as e:
                self.logger.error(f"Failed to send to {connection_id}: {e}")
                await self.remove_connection(connection_id)
        return False

    def get_connection_ids(self) -> List[str]:
        """Get list of all connection IDs."""
        return list(self.connections.keys())

    def get_connection_count(self) -> int:
        """Get number of active connections."""
        return len(self.connections)

    async def close_all(self) -> None:
        """Close all connections in the pool."""
        for websocket in list(self.connections.values()):
            try:
                await websocket.close()
            except Exception:
                pass
        self.connections.clear()

add_connection(connection_id, websocket) async

Add a WebSocket connection to the pool.

Source code in toolboxv2/mods/WebSocketManager.py

async def add_connection(self, connection_id: str, websocket: Any) -> None:
    """Add a WebSocket connection to the pool."""
    self.connections[connection_id] = websocket
    self.logger.info(f"Added connection {connection_id} (total: {len(self.connections)})")

broadcast(message, exclude_connection=None) async

Broadcast message to all connections in the pool.

Source code in toolboxv2/mods/WebSocketManager.py

async def broadcast(self, message: str, exclude_connection: Optional[str] = None) -> int:
    """Broadcast message to all connections in the pool."""
    sent_count = 0
    for conn_id, websocket in list(self.connections.items()):
        if conn_id != exclude_connection:
            try:
                await websocket.send(message)
                sent_count += 1
            except Exception as e:
                self.logger.error(f"Failed to send to {conn_id}: {e}")
                await self.remove_connection(conn_id)
    return sent_count

close_all() async

Close all connections in the pool.

Source code in toolboxv2/mods/WebSocketManager.py

async def close_all(self) -> None:
    """Close all connections in the pool."""
    for websocket in list(self.connections.values()):
        try:
            await websocket.close()
        except Exception:
            pass
    self.connections.clear()

get_connection_count()

Get number of active connections.

Source code in toolboxv2/mods/WebSocketManager.py

def get_connection_count(self) -> int:
    """Get number of active connections."""
    return len(self.connections)

get_connection_ids()

Get list of all connection IDs.

Source code in toolboxv2/mods/WebSocketManager.py

def get_connection_ids(self) -> List[str]:
    """Get list of all connection IDs."""
    return list(self.connections.keys())

handle_message(connection_id, message) async

Route incoming messages to appropriate handlers.

Source code in toolboxv2/mods/WebSocketManager.py

async def handle_message(self, connection_id: str, message: str) -> None:
    """Route incoming messages to appropriate handlers."""
    try:
        ws_message = WebSocketMessage.from_json(message)
        action = ws_message.event

        # Handle ping/pong
        if action == 'ping':
            pong_message = WebSocketMessage(event='pong', data={})
            await self.send_to_connection(connection_id, pong_message.to_json())
            return

        # Try global actions first
        if action in self.global_actions:
            # Run in executor to prevent blocking
            loop = asyncio.get_event_loop()
            await loop.run_in_executor(
                None,
                lambda: asyncio.create_task(
                    self.global_actions[action](self.pool_id, connection_id, ws_message)
                )
            )
        # Then try connection-specific actions
        elif connection_id in self.actions and action in self.actions[connection_id]:
            loop = asyncio.get_event_loop()
            await loop.run_in_executor(
                None,
                lambda: asyncio.create_task(
                    self.actions[connection_id][action](self.pool_id, connection_id, ws_message)
                )
            )
        else:
            self.logger.warning(f"No handler for action '{action}' from {connection_id}")

    except json.JSONDecodeError:
        self.logger.error(f"Invalid JSON from {connection_id}: {message[:100]}")
    except Exception as e:
        self.logger.error(f"Error handling message from {connection_id}: {e}")

register_action(action_name, handler, connection_ids=None)

Register an action handler for specific connections or globally.

Source code in toolboxv2/mods/WebSocketManager.py

def register_action(self, action_name: str, handler: Callable,
                    connection_ids: Optional[List[str]] = None) -> None:
    """Register an action handler for specific connections or globally."""
    if connection_ids is None:
        self.global_actions[action_name] = handler
        self.logger.info(f"Registered global action: {action_name}")
    else:
        for conn_id in connection_ids:
            if conn_id not in self.actions:
                self.actions[conn_id] = {}
            self.actions[conn_id][action_name] = handler
        self.logger.info(f"Registered action {action_name} for connections: {connection_ids}")

remove_connection(connection_id) async

Remove a WebSocket connection from the pool.

Source code in toolboxv2/mods/WebSocketManager.py

async def remove_connection(self, connection_id: str) -> None:
    """Remove a WebSocket connection from the pool."""
    if connection_id in self.connections:
        del self.connections[connection_id]
        self.logger.info(f"Removed connection {connection_id} (remaining: {len(self.connections)})")

send_to_connection(connection_id, message) async

Send message to a specific connection.

Source code in toolboxv2/mods/WebSocketManager.py

async def send_to_connection(self, connection_id: str, message: str) -> bool:
    """Send message to a specific connection."""
    if connection_id in self.connections:
        try:
            await self.connections[connection_id].send(message)
            return True
        except Exception as e:
            self.logger.error(f"Failed to send to {connection_id}: {e}")
            await self.remove_connection(connection_id)
    return False

WebSocketServer

WebSocket server with pool management.

Source code in toolboxv2/mods/WebSocketManager.py

class WebSocketServer:
    """WebSocket server with pool management."""

    def __init__(self, host: str = "localhost", port: int = 8765):
        self.host = host
        self.port = port
        self.pools: Dict[str, WebSocketPool] = {}
        self.server = None
        self.logger = logging.getLogger("WSServer")

    def create_pool(self, pool_id: str) -> WebSocketPool:
        """Create a new connection pool."""
        if pool_id not in self.pools:
            self.pools[pool_id] = WebSocketPool(pool_id)
            self.logger.info(f"Created pool: {pool_id}")
        return self.pools[pool_id]

    def get_pool(self, pool_id: str) -> Optional[WebSocketPool]:
        """Get an existing pool."""
        return self.pools.get(pool_id)

    async def handle_connection(self, websocket, path: str):
        """Handle new WebSocket connections."""
        connection_id = f"conn_{id(websocket)}"
        pool_id = path.strip('/') or 'default'

        pool = self.create_pool(pool_id)
        await pool.add_connection(connection_id, websocket)

        self.logger.info(f"New connection {connection_id} in pool {pool_id}")

        try:
            # Ping-Task für diese Verbindung starten
            ping_task = asyncio.create_task(self._connection_ping_loop(websocket, connection_id))

            async for message in websocket:
                # Message handling in background to prevent blocking
                asyncio.create_task(pool.handle_message(connection_id, message))

        except ConnectionClosed:
            self.logger.info(f"Connection {connection_id} closed normally")
        except Exception as e:
            self.logger.error(f"Connection error for {connection_id}: {e}")
        finally:
            ping_task.cancel()
            await pool.remove_connection(connection_id)

    async def _connection_ping_loop(self, websocket, connection_id: str):
        """Ping loop for individual connection."""
        try:
            while not websocket.closed:
                await asyncio.sleep(30)  # Ping every 30 seconds
                await websocket.ping()
        except Exception as e:
            self.logger.debug(f"Ping loop ended for {connection_id}: {e}")

    async def start(self, non_blocking: bool = False) -> None:
        """Start the WebSocket server."""
        if non_blocking is None:
            return
        self.server = await ws_serve(self.handle_connection, self.host, self.port)
        self.logger.info(f"🚀 WebSocket server started on {self.host}:{self.port}")

        if not non_blocking:
            await self.server.wait_closed()

    async def stop(self) -> None:
        """Stop the server and close all connections."""
        if self.server:
            self.server.close()
            await self.server.wait_closed()

        # Close all pools
        for pool in self.pools.values():
            await pool.close_all()
        self.pools.clear()

        self.logger.info("✅ Server stopped")

create_pool(pool_id)

Create a new connection pool.

Source code in toolboxv2/mods/WebSocketManager.py

def create_pool(self, pool_id: str) -> WebSocketPool:
    """Create a new connection pool."""
    if pool_id not in self.pools:
        self.pools[pool_id] = WebSocketPool(pool_id)
        self.logger.info(f"Created pool: {pool_id}")
    return self.pools[pool_id]

get_pool(pool_id)

Get an existing pool.

Source code in toolboxv2/mods/WebSocketManager.py

def get_pool(self, pool_id: str) -> Optional[WebSocketPool]:
    """Get an existing pool."""
    return self.pools.get(pool_id)

handle_connection(websocket, path) async

Handle new WebSocket connections.

Source code in toolboxv2/mods/WebSocketManager.py

async def handle_connection(self, websocket, path: str):
    """Handle new WebSocket connections."""
    connection_id = f"conn_{id(websocket)}"
    pool_id = path.strip('/') or 'default'

    pool = self.create_pool(pool_id)
    await pool.add_connection(connection_id, websocket)

    self.logger.info(f"New connection {connection_id} in pool {pool_id}")

    try:
        # Ping-Task für diese Verbindung starten
        ping_task = asyncio.create_task(self._connection_ping_loop(websocket, connection_id))

        async for message in websocket:
            # Message handling in background to prevent blocking
            asyncio.create_task(pool.handle_message(connection_id, message))

    except ConnectionClosed:
        self.logger.info(f"Connection {connection_id} closed normally")
    except Exception as e:
        self.logger.error(f"Connection error for {connection_id}: {e}")
    finally:
        ping_task.cancel()
        await pool.remove_connection(connection_id)

start(non_blocking=False) async

Start the WebSocket server.

Source code in toolboxv2/mods/WebSocketManager.py

async def start(self, non_blocking: bool = False) -> None:
    """Start the WebSocket server."""
    if non_blocking is None:
        return
    self.server = await ws_serve(self.handle_connection, self.host, self.port)
    self.logger.info(f"🚀 WebSocket server started on {self.host}:{self.port}")

    if not non_blocking:
        await self.server.wait_closed()

stop() async

Stop the server and close all connections.

Source code in toolboxv2/mods/WebSocketManager.py

async def stop(self) -> None:
    """Stop the server and close all connections."""
    if self.server:
        self.server.close()
        await self.server.wait_closed()

    # Close all pools
    for pool in self.pools.values():
        await pool.close_all()
    self.pools.clear()

    self.logger.info("✅ Server stopped")

WhatsAppTb

client

DocumentSystem

Source code in toolboxv2/mods/WhatsAppTb/client.py

class DocumentSystem:
    def __init__(self, storage: BlobStorage):
        self.storage = storage
        self.media_types = {
            'document': ['pdf', 'doc', 'docx', 'txt'],
            'image': ['jpg', 'jpeg', 'png', 'gif'],
            'video': ['mp4', 'mov', 'avi']
        }

    def list_documents(self, filter_type: str = None) -> list[dict]:
        """List all documents with metadata"""
        docs = []
        for blob_id in self.storage._get_all_blob_ids():
            with BlobFile(blob_id, 'r', self.storage) as f:
                metadata = f.read_json()
                if metadata:
                    docs.append({
                        'id': blob_id,
                        'name': metadata.get('filename', blob_id),
                        'type': metadata.get('type', 'document'),
                        'size': metadata.get('size', 0),
                        'modified': metadata.get('timestamp', ''),
                        'preview': metadata.get('preview', '')
                    })
        if filter_type:
            return [d for d in docs if d['type'] == filter_type]
        return docs

    def save_document(self, file_data: bytes, filename: str, file_type: str) -> str:
        """Save a document with metadata"""
        blob_id = self.storage._generate_blob_id()
        metadata = {
            'filename': filename,
            'type': file_type,
            'size': len(file_data),
            'timestamp': datetime.now().isoformat(),
            'preview': self._generate_preview(file_data, file_type)
        }

        with BlobFile(blob_id, 'w', self.storage) as f:
            f.write_json(metadata)
            f.write(file_data)
        return blob_id

    def delete_document(self, blob_id: str) -> bool:
        """Delete a document"""
        try:
            self.storage.delete_blob(blob_id)
            return True
        except Exception as e:
            logging.error(f"Delete failed: {str(e)}")
            return False

    def search_documents(self, query: str) -> list[dict]:
        """Search documents by filename or content"""
        results = []
        for doc in self.list_documents():
            if query.lower() in doc['name'].lower() or self._search_in_content(doc['id'], query):
                results.append(doc)
        return results

    def _generate_preview(self, data: bytes, file_type: str) -> str:
        """Generate preview based on file type"""
        if file_type in self.media_types['image']:
            return f"Image preview: {data[:100].hex()}"
        elif file_type in self.media_types['video']:
            return "Video preview unavailable"
        return data[:100].decode('utf-8', errors='ignore')

    def _search_in_content(self, blob_id: str, query: str) -> bool:
        """Search content within documents"""
        try:
            with BlobFile(blob_id, 'r', self.storage) as f:
                content = f.read().decode('utf-8', errors='ignore')
                return query.lower() in content.lower()
        except:
            return False

delete_document(blob_id)

Delete a document

Source code in toolboxv2/mods/WhatsAppTb/client.py

def delete_document(self, blob_id: str) -> bool:
    """Delete a document"""
    try:
        self.storage.delete_blob(blob_id)
        return True
    except Exception as e:
        logging.error(f"Delete failed: {str(e)}")
        return False

list_documents(filter_type=None)

List all documents with metadata

Source code in toolboxv2/mods/WhatsAppTb/client.py

def list_documents(self, filter_type: str = None) -> list[dict]:
    """List all documents with metadata"""
    docs = []
    for blob_id in self.storage._get_all_blob_ids():
        with BlobFile(blob_id, 'r', self.storage) as f:
            metadata = f.read_json()
            if metadata:
                docs.append({
                    'id': blob_id,
                    'name': metadata.get('filename', blob_id),
                    'type': metadata.get('type', 'document'),
                    'size': metadata.get('size', 0),
                    'modified': metadata.get('timestamp', ''),
                    'preview': metadata.get('preview', '')
                })
    if filter_type:
        return [d for d in docs if d['type'] == filter_type]
    return docs

save_document(file_data, filename, file_type)

Save a document with metadata

Source code in toolboxv2/mods/WhatsAppTb/client.py

def save_document(self, file_data: bytes, filename: str, file_type: str) -> str:
    """Save a document with metadata"""
    blob_id = self.storage._generate_blob_id()
    metadata = {
        'filename': filename,
        'type': file_type,
        'size': len(file_data),
        'timestamp': datetime.now().isoformat(),
        'preview': self._generate_preview(file_data, file_type)
    }

    with BlobFile(blob_id, 'w', self.storage) as f:
        f.write_json(metadata)
        f.write(file_data)
    return blob_id

search_documents(query)

Search documents by filename or content

Source code in toolboxv2/mods/WhatsAppTb/client.py

def search_documents(self, query: str) -> list[dict]:
    """Search documents by filename or content"""
    results = []
    for doc in self.list_documents():
        if query.lower() in doc['name'].lower() or self._search_in_content(doc['id'], query):
            results.append(doc)
    return results

WhatsAppAssistant dataclass

Source code in toolboxv2/mods/WhatsAppTb/client.py

@dataclass
class WhatsAppAssistant:
    whc: WhClient
    isaa: 'Tools'
    agent: Optional['Agent'] = None
    credentials: Credentials | None = None
    state: AssistantState = AssistantState.OFFLINE

    # Service clients
    gmail_service: Any = None
    calendar_service: Any = None

    start_time: Any = None

    blob_docs_system: Any = None
    duration_minutes: int = 20
    credentials_path: str = "/root/Toolboxv2/credentials.json"
    # Progress messengers
    progress_messengers: dict[str, 'ProgressMessenger'] = field(default_factory=dict)
    buttons: dict[str, dict] = field(default_factory=dict)
    history: FileCache = field(default_factory=FileCache)

    pending_actions: dict[str, dict] = field(default_factory=dict)


    def __post_init__(self):

        self.start_time = datetime.now()
        self.processed_messages = set()
        self.message_lock = threading.Lock()
        self.audio_processor = None
        self.blob_docs_system = DocumentSystem(BlobStorage())
        self.stt = get_app().run_any(TBEF.AUDIO.STT_GENERATE,
                                     model="openai/whisper-small",
                                     row=False, device=1)

        self.pending_actions[self.whc.progress_messenger0.recipient_phone] = {}

        self.load_credentials()
        self.setup_progress_messengers()
        self.setup_interaction_buttons()
        self.history = FileCache(folder=".data/WhatsAppAssistant")
        self.state = AssistantState.ONLINE

    async def generate_authorization_url(self, *a):
        """
        Generate an authorization URL for user consent

        :return: Authorization URL for the user to click and authorize access
        """
        from google_auth_oauthlib.flow import Flow
        # Define the scopes required for Gmail and Calendar
        SCOPES = [
            'https://www.googleapis.com/auth/gmail.modify',
            'https://www.googleapis.com/auth/calendar'
        ]

        # Create a flow instance to manage the OAuth 2.0 authorization process
        flow = Flow.from_client_secrets_file(
            self.credentials_path,
            scopes=SCOPES,
            redirect_uri='urn:ietf:wg:oauth:2.0:oob'  # Use 'urn:ietf:wg:oauth:2.0:oob' for desktop apps
        )

        # Generate the authorization URL
        authorization_url, _ = flow.authorization_url(
            access_type='offline',  # Allows obtaining refresh token
            prompt='consent'  # Ensures user is always prompted for consent
        )
        self.pending_actions[self.whc.progress_messenger0.recipient_phone] = {'type': 'auth',
                                                                              'step': 'awaiting_key'}
        return {
            'type': 'quick_reply',
            'text': f'Url to log in {authorization_url}',
            'options': {'cancel': '❌ Cancel Upload'}
        }

    def complete_authorization(self, message: Message):
        """
        Complete the authorization process using the authorization code

        :param authorization_code: Authorization code received from Google
        """
        from google_auth_oauthlib.flow import Flow
        authorization_code = message.content
        # Define the scopes required for Gmail and Calendar
        SCOPES = [
            'https://www.googleapis.com/auth/gmail.modify',
            'https://www.googleapis.com/auth/calendar'
        ]

        # Create a flow instance to manage the OAuth 2.0 authorization process
        flow = Flow.from_client_secrets_file(
            self.credentials_path,
            scopes=SCOPES,
            redirect_uri='urn:ietf:wg:oauth:2.0:oob'
        )

        # Exchange the authorization code for credentials
        flow.fetch_token(code=authorization_code)
        self.credentials = flow.credentials

        # Save the credentials for future use
        self.save_credentials()

        # Initialize services
        self.init_services()
        return "Done"


    def save_credentials(self):
        """
        Save the obtained credentials to a file for future use
        """
        if not os.path.exists('token'):
            os.makedirs('token')

        with open('token/google_token.json', 'w') as token_file:
            token_file.write(self.credentials.to_json())


    def load_credentials(self):
        """
        Load previously saved credentials if available

        :return: Whether credentials were successfully loaded
        """
        try:
            self.credentials = Credentials.from_authorized_user_file('token/google_token.json')
            self.init_services()
            return True
        except FileNotFoundError:
            return False


    def init_services(self):
        """
        Initialize Gmail and Calendar services
        """
        from googleapiclient.discovery import build

        self.gmail_service = build('gmail', 'v1', credentials=self.credentials)
        self.calendar_service = build('calendar', 'v3', credentials=self.credentials)
        self.pending_actions[self.whc.progress_messenger0.recipient_phone] = {}

    def setup_progress_messengers(self):
        """Initialize progress messengers for different types of tasks"""
        self.progress_messengers = {
            'task': self.whc.progress_messenger0,
            'email': self.whc.progress_messenger1,
            'calendar': self.whc.progress_messenger2
        }

    def setup_interaction_buttons(self):
        """Define WhatsApp interaction buttons for different functionalities"""
        self.buttons = {
            'menu': {
                'header': 'Digital Assistant',
                'body': 'Please select an option:',
                'footer': '-- + --',
                'action': {
                    'button': 'Menu',
                    'sections': [
                        {
                            'title': 'Main Functions',
                            'rows': [
                                {'id': 'agent', 'title': 'Agent Controls', 'description': 'Manage your AI assistant'},
                                {'id': 'email', 'title': 'Email Management', 'description': 'Handle your emails'},
                                {'id': 'calendar', 'title': 'Calendar', 'description': 'Manage your schedule'},
                                {'id': 'docs', 'title': 'Documents', 'description': 'Handle documents'},
                                {'id': 'system', 'title': 'System', 'description': 'System controls and metrics'}
                            ]
                        }
                    ]
                }
            },
            'agent': self._create_agent_controls_buttons(),
            'email': self._create_email_controls_buttons(),
            'calendar': self._create_calendar_controls_buttons(),
            'docs': self._create_docs_controls_buttons(),
            'system': self._create_system_controls_buttons()
        }

    @staticmethod
    def _create_agent_controls_buttons():
        return {
            'header': 'Agent Controls',
            'body': 'Manage your AI assistant:',
            'action': {
                'button': 'Select',
                'sections': [
                    {
                        'title': 'Basic Actions',
                        'rows': [
                            {'id': 'agent-task', 'title': 'Agent Task', 'description': 'Run the agent'},
                            {'id': 'start', 'title': 'Start Agent', 'description': 'Run taskstack in background'},
                            {'id': 'stop', 'title': 'Stop Agent', 'description': 'Stop taskstack execution'}
                        ]
                    },
                    {
                        'title': 'Advanced Actions',
                        'rows': [
                            {'id': 'system-task', 'title': 'System Task',
                             'description': 'Run the Isaa Reasoning Agent system'},
                            {'id': 'tasks', 'title': 'Task Stack', 'description': 'View and manage tasks'},
                            {'id': 'memory', 'title': 'Clear Memory', 'description': 'Reset agent memory'}
                        ]
                    }
                ]
            }
        }

    @staticmethod
    def _create_email_controls_buttons():
        return {
            'header': 'Email Management',
            'body': 'Handle your emails:',
            'action': {
                'button': 'Select',
                'sections': [
                    {
                        'title': 'Basic Actions',
                        'rows': [
                            {'id': 'check', 'title': 'Check Emails', 'description': 'View recent emails'},
                            {'id': 'send', 'title': 'Send Email', 'description': 'Compose new email'},
                            {'id': 'summary', 'title': 'Get Summary', 'description': 'Summarize emails'}
                        ]
                    },
                    {
                        'title': 'Advanced Actions',
                        'rows': [
                            {'id': 'search', 'title': 'Search', 'description': 'Search emails'}
                        ]
                    }
                ]
            }
        }

    @staticmethod
    def _create_calendar_controls_buttons():
        return {
            'header': 'Calendar Management',
            'body': 'Manage your schedule:',
            'action': {
                'button': 'Select',
                'sections': [
                    {
                        'title': 'Basic Actions',
                        'rows': [
                            {'id': 'today', 'title': 'Today\'s Events', 'description': 'View today\'s schedule'},
                            {'id': 'add', 'title': 'Add Event', 'description': 'Create new event'},
                            {'id': 'upcoming', 'title': 'Upcoming', 'description': 'View upcoming events'}
                        ]
                    },
                    {
                        'title': 'Advanced Actions',
                        'rows': [
                            {'id': 'find_slot', 'title': 'Find Time Slot', 'description': 'Find available time'}
                        ]
                    }
                ]
            }
        }

    @staticmethod
    def _create_docs_controls_buttons():
        return {
            'header': 'Document Management',
            'body': 'Handle your documents:',
            'action': {
                'button': 'Select',
                'sections': [
                    {
                        'title': 'Basic Actions',
                        'rows': [
                            {'id': 'upload', 'title': 'Upload', 'description': 'Add new document'},
                            {'id': 'list', 'title': 'List Documents', 'description': 'View all documents'},
                            {'id': 'search', 'title': 'Search', 'description': 'Search documents'}
                        ]
                    },
                    {
                        'title': 'Advanced Actions',
                        'rows': [
                            {'id': 'delete', 'title': 'Delete', 'description': 'Remove document'}
                        ]
                    }
                ]
            }
        }

    @staticmethod
    def _create_system_controls_buttons():
        return {
            'header': 'System Controls',
            'body': 'System management:',
            'action': {
                'button': 'Select',
                'sections': [
                    {
                        'title': 'Basic Actions',
                        'rows': [
                            {'id': 'status', 'title': 'System Status', 'description': 'View current status'},
                            {'id': 'restart', 'title': 'Restart', 'description': 'Restart system'},
                            {'id': 'connect', 'title': 'Connect', 'description': 'Connect to Google Calendar and Email'}
                        ]
                    }
                ]
            }
        }

    async def handle_message(self, message: 'Message'):
        """Main message handler for incoming WhatsApp messages"""

        # Deduplication check
        with self.message_lock:
            if message.id in self.processed_messages:
                return
            last_ts = time.time()
            print(last_ts)
            if len(self.processed_messages) > 0:
                m_id, last_ts = self.processed_messages.pop()
                self.processed_messages.add((m_id, last_ts))

            print("DUPLICATION P", message.data.get('entry', [{}])[0].get('changes', [{}])[0].get('value', {}).get('messages', [{}])[0].get('timestamp', 0) , last_ts)
            if float(message.data.get('entry', [{}])[0].get('changes', [{}])[0].get('value', {}).get('messages', [{}])[0].get('timestamp', 0)) < last_ts - 120:
                return
            self.processed_messages.add((message.id, time.perf_counter()))

        # Mark message as read
        message.mark_as_read()

        # Extract content and type
        content_type = message.type
        content = message.content

        print(f"message.content {content=} {content_type=} {message.data=}")

        try:
            if content_type == 'interactive':
                await self.handle_interactive(message)
            elif content_type == 'audio':
                await self.handle_audio_message(message)
            elif content_type in ['document', 'image', 'video']:
                response = await self.handle_media_message(message)
                self.save_reply(message, response)
            elif content_type == 'text':
                if content.lower() == "menu":
                    self.whc.messenger.send_button(
                        recipient_id=self.whc.progress_messenger0.recipient_phone,
                        button=self.buttons[content.lower()]
                    )
                else:
                    await self.helper_text(message)
            else:
                message.reply("Unsupported message type")
        #except Exception as e:
        #    logging.error(f"Message handling error: {str(e)}")
        #   message.reply("❌ Error processing request")
        finally:
            # Cleanup old messages (keep 1 hour history)
            with self.message_lock:
                self._clean_processed_messages()

    async def helper_text(self, message: 'Message', return_text=False):
        if not isinstance(message.content, str) and not len(message.content) > 0:
            content = self.whc.messenger.get_message(message.data)
            print(f"contents {content=}, {message.content=}")
            message.content = content
        self.history.set(message.id, message.content)
        if len(self.pending_actions[self.whc.progress_messenger0.recipient_phone].keys()) != 0:
            message.reply(
                f"Open Interaction : {json.dumps(self.pending_actions[self.whc.progress_messenger0.recipient_phone], indent=2)}")
            if self.pending_actions[self.whc.progress_messenger0.recipient_phone].get('type') == 'auth':
                res = self.complete_authorization(message)
                self.save_reply(message, res)
            res = await self.handle_calendar_actions(message)
            if res:
                self.save_reply(message, res)
                return
            res2 = await self.handle_email_actions(message)
            if res2:
                self.save_reply(message, res2)
                return
            await self.handle_agent_actions(message)
            return
        await self.handle_agent_actions(message)

    async def handle_interactive(self, message: Message):
        """Handle all interactive messages"""
        content = self.whc.messenger.get_interactive_response(message.data)
        if content.get("type") == "list_reply":
            await self.handle_button_interaction(content.get("list_reply"), message)
        elif content.get("type") == "button_reply":
            print(content)

    async def handle_audio_message(self, message: 'Message'):
        """Process audio messages with STT and TTS"""
        # Download audio
        progress = self.progress_messengers['task']
        stop_flag = threading.Event()
        # message_id = progress.send_initial_message(mode="loading")
        progress.message_id = message.id
        progress.start_loading_in_background(stop_flag)

        content = self.whc.messenger.get_audio(message.data)
        audio_file_name = self.whc.messenger.download_media(media_url=self.whc.messenger.query_media_url(media_id=content.get('id')), mime_type='audio/opus', file_path=".data/temp")
        print(f"audio_file_name {audio_file_name}")
        if audio_file_name is None:
            message.reply("Could not process audio file")
            stop_flag.set()
            return

        text = self.stt(audio_file_name)['text']
        if not text:
            message.reply("Could not process audio")
            stop_flag.set()
            return

        message.reply("Transcription :\n "+ text)
        message.content = text
        agent_res = await self.helper_text(message, return_text=True)

        if agent_res is not None:
            pass

        stop_flag.set()
        # Process text and get response
        # response = await self.process_input(text, message)

        # Convert response to audio
        #audio_file = self.audio_processor.tts(response)
        #audio_file = None # TODO
        #self.whc.messenger.send_audio(
        #    audio=audio_file,
        #    recipient_id=self.whc.progress_messenger0.recipient_phone,
        #)

    async def confirm(self, message: Message):
        status = self.pending_actions[self.whc.progress_messenger0.recipient_phone]
        if status.get('type') == "create_event":
            if status.get('step') == "confirm_envet":
                event = self._create_calendar_event(status.get('event_data'))
                self.pending_actions[self.whc.progress_messenger0.recipient_phone] = {}
                return f"✅ Event created!\n{event.get('htmlLink')}"
            return "❌"
        elif status.get('type') == "compose_email":
            if status.get('step') == "confirm_email":
                # Send email
                result = self.gmail_service.users().messages().send(
                    userId='me',
                    body=self._build_email_draft(status['draft'])
                ).execute()
                self.pending_actions[self.whc.progress_messenger0.recipient_phone] = {}
                return f"✅ Email sent! Message ID: {result['id']}"
            return "❌"
        return "❌ Done"

    async def cancel(self, *a):
        self.pending_actions[self.whc.progress_messenger0.recipient_phone] = {}
        return "✅ cancel Done"

    async def handle_button_interaction(self, content: dict, message: Message):
        """Handle button click interactions"""
        button_id = content['id']

        # First check if it's a main menu button
        if button_id in self.buttons:
            self.whc.messenger.send_button(
                recipient_id=self.whc.progress_messenger0.recipient_phone,
                button=self.buttons[button_id]
            )
            return

        # Handle action buttons
        action_handlers = {
            # Agent controls
            'start': self.start_agent,
            'stop': self.stop_agent,
            'tasks': self.show_task_stack,
            'memory': self.clear_memory,
            'system-task': self.system_task,
            'agent-task': self.agent_task,

            # Email controls
            'check': self.check_emails,
            'send': self.start_email_compose,
            'summary': self.email_summary,
            'search': self.email_search,

            # Calendar controls
            'today': self.show_today_events,
            'add': self.start_event_create,
            'upcoming': self.show_upcoming_events,
            'find_slot': self.find_time_slot,

            # Document controls
            'upload': self.start_document_upload,
            'list': self.list_documents,
            'search_docs': self.search_documents,
            'delete': self.delete_document,

            # System controls
            'status': self.system_status,
            'restart': self.restart_system,
            'connect': self.generate_authorization_url,

            'cancel': self.cancel,
            'confirm': self.confirm,
        }
        if button_id in action_handlers:
            try:
                # Start progress indicator
                progress = self.progress_messengers['task']
                stop_flag = threading.Event()
                # message_id = progress.send_initial_message(mode="loading")
                progress.message_id = message.id
                progress.start_loading_in_background(stop_flag)

                # Execute handler

                result = await action_handlers[button_id](message)


                # Send result
                if isinstance(result, str):
                    self.save_reply(message, result)
                elif isinstance(result, dict):  # For structured responses
                    self.send_structured_response(result)

                stop_flag.set()
            finally:
                #except Exception as e:
                stop_flag.set()
            #    message.reply(f"❌ Error processing {button_id}: {str(e)}")
        elif 'event_' in button_id:
            res = await self.get_event_details(button_id.replace("event_", ''))
            if isinstance(res, str):
                self.save_reply(message, res)
                return
            for r in res:
                if isinstance(r, str):
                    self.save_reply(message, r)
                else:
                    self.whc.messenger.send_location(**r)

        elif 'email_' in button_id:
            res = await self.get_email_details(button_id.replace("email_", ''))
            self.save_reply(message, res)
        else:
            message.reply("⚠️ Unknown command")

    def send_structured_response(self, result: dict):
        """Send complex responses using appropriate WhatsApp features"""
        if result['type'] == 'list':
            self.whc.messenger.send_button(
                recipient_id=self.whc.progress_messenger0.recipient_phone,
                button={
                    'header': result.get('header', ''),
                    'body': result.get('body', ''),
                    'footer': result.get('footer', ''),
                    'action': {
                        'button': 'Action',
                        'sections': result['sections']
                    }
                }
            )
        elif result['type'] == 'quick_reply':
            self.whc.messenger.send_button(
                recipient_id=self.whc.progress_messenger0.recipient_phone,
                button={
                    'header': "Quick reply",
                    'body': result['text'],
                    'footer': '',
                    'action': {'button': 'Action', 'sections': [{
                        'title': 'View',
                        'rows': [{'id': k, 'title': v[:23]} for k, v in result['options'].items()]
                    }]}
                }
            )

        elif result['type'] == 'media':
            if result['media_type'] == 'image':
                self.whc.messenger.send_image(
                    image=result['url'],
                    recipient_id=self.whc.progress_messenger0.recipient_phone,
                    caption=result.get('caption', '')
                )
            elif result['media_type'] == 'document':
                self.whc.messenger.send_document(
                    document=result['url'],
                    recipient_id=self.whc.progress_messenger0.recipient_phone,
                    caption=result.get('caption', '')
                )

    async def clear_memory(self, message):
        self.agent.reset_context()
        self.agent.taskstack.tasks = []
        return "🧠 Memory cleared successfully"

    async def system_task(self, message):
        """Initiate email search workflow"""
        self.pending_actions[self.whc.progress_messenger0.recipient_phone] = {
            'type': 'system',
            'step': 'await_query'
        }
        return {
            'type': 'quick_reply',
            'text': "Now prompt the 🧠ISAA-System 📝",
            'options': {'cancel': '❌ Cancel Search'}
        }

    async def agent_task(self, message):
        """Initiate email search workflow"""
        self.pending_actions[self.whc.progress_messenger0.recipient_phone] = {
            'type': 'self-agent',
            'step': 'await_query'
        }
        return {
            'type': 'quick_reply',
            'text': "Now prompt the self-agent 📝",
            'options': {'cancel': '❌ Cancel Search'}
        }

    async def check_emails(self, message, query=""):
        """Improved email checking with WhatsApp API formatting"""
        if not self.gmail_service:
            return "⚠️ Gmail service not configured"

        try:
            results = self.gmail_service.users().messages().list(
                userId='me',
                maxResults=10,
                labelIds=['INBOX'],
                q=query
            ).execute()

            emails = []
            for msg in results.get('messages', [])[:10]:
                email_data = self.gmail_service.users().messages().get(
                    userId='me',
                    id=msg['id'],
                    format='metadata'
                ).execute()

                headers = {h['name']: h['value'] for h in email_data['payload']['headers']}
                emails.append({
                    'id': msg['id'],
                    'from': headers.get('From', 'Unknown'),
                    'subject': headers.get('Subject', 'No Subject'),
                    'date': headers.get('Date', 'Unknown'),
                    'snippet': email_data.get('snippet', ''),
                    'unread': 'UNREAD' in email_data.get('labelIds', [])
                })

            return {
                'type': 'list',
                'header': '📨 Recent Emails',
                'body': 'Tap to view full email',
                'footer': 'Email Manager',
                'sections': [{
                    'title': f"Inbox ({len(emails)} emails)",
                    'rows': [{
                        'id': f"email_{email['id']}",
                        'title': f"{'📬' if email['unread'] else '📭'} {email['subject']}"[:23],
                        'description': f"From: {email['from']}\n{email['snippet']}"[:45]
                    } for email in emails]
                }]
            }
        except Exception as e:
            return f"⚠️ Error fetching emails: {str(e)}"

    async def get_email_details(self, email_id):
        """Retrieve and format full email details"""
        if not self.gmail_service:
            return "⚠️ Gmail service not configured"

        try:
            email_data = self.gmail_service.users().messages().get(
                userId='me',
                id=email_id,
                format='full'
            ).execute()

            headers = {h['name']: h['value'] for h in email_data['payload']['headers']}
            body = ""
            for part in email_data.get('payload', {}).get('parts', []):
                if part['mimeType'] == 'text/plain':
                    body = base64.urlsafe_b64decode(part['body']['data']).decode('utf-8')
                    break

            formatted_text = (
                f"📧 *Email Details*\n\n"
                f"From: {headers.get('From', 'Unknown')}\n"
                f"Subject: {headers.get('Subject', 'No Subject')}\n"
                f"Date: {headers.get('Date', 'Unknown')}\n\n"
                f"{body[:15000]}{'...' if len(body) > 15000 else ''}"
            )
            return  self.agent.mini_task(
                formatted_text , "system", "Summarize the email in bullet points with key details"
            )
        except Exception as e:
            return f"⚠️ Error fetching email: {str(e)}"

    async def email_summary(self, message):
        """Generate AI-powered email summaries"""
        try:
            messages = self.gmail_service.users().messages().list(
                userId='me',
                maxResults=3,
                labelIds=['INBOX']
            ).execute().get('messages', [])

            email_contents = []
            for msg in messages[:3]:
                email_data = self.gmail_service.users().messages().get(
                    userId='me',
                    id=msg['id'],
                    format='full'
                ).execute()
                email_contents.append(self._parse_email_content(email_data))

            summary = self.agent.mini_task(
                "\n\n".join(email_contents) , "system", "Summarize these emails in bullet points with key details:"
            )

            return f"📋 Email Summary:\n{summary}\n\n*Powered by AI*"
        except Exception as e:
            logging.error(f"Summary failed: {str(e)}")
            return f"❌ Could not generate summary: {str(e)}"

    async def email_search(self, message):
        """Initiate email search workflow"""
        self.pending_actions[self.whc.progress_messenger0.recipient_phone] = {
            'type': 'email_search',
            'step': 'await_query'
        }
        return {
            'type': 'quick_reply',
            'text': "🔍 What would you like to search for?",
            'options': {'cancel': '❌ Cancel Search'}
        }

    async def start_email_compose(self, message):
        """Enhanced email composition workflow"""
        self.pending_actions[self.whc.progress_messenger0.recipient_phone] = {
            'type': 'compose_email',
            'step': 'subject',
            'draft': {'attachments': []}
        }
        return {
            'type': 'quick_reply',
            'text': "📝 Let's compose an email\n\nSubject:",
            'options': {'cancel': '❌ Cancel Composition'}
        }

    async def handle_email_actions(self, message):
        """Handle multi-step email workflows"""
        user_state = self.pending_actions.get(self.whc.progress_messenger0.recipient_phone, {})

        if user_state.get('type') == 'compose_email':
            return await self._handle_email_composition(message, user_state)
        if user_state.get('type') == 'email_search':
            return await self.check_emails(message, self.agent.mini_task("""Conventire Pezise zu einer googel str only query using : Gmail Suchoperatoren!

Basis-Operatoren:
- from: Absender
- to: Empfänger
- subject: Betreff
- label: Gmail Label
- has:attachment Anhänge
- newer_than:7d Zeitfilter
- before: Datum vor
- after: Datum nach

Erweiterte Operatoren:
- in:inbox
- in:sent
- in:spam
- cc: Kopie
- bcc: Blindkopie
- is:unread
- is:read
- larger:10M Größenfilter
- smaller:5M
- filename:pdf Dateityp

Profi-Tipps:
- Kombinierbar mit UND/ODER
- Anführungszeichen für exakte Suche
- Negation mit -
 beispeile : 'Ungelesene Mails letzte Woche': -> 'is:unread newer_than:7d'

""", "user",message.content))


        return None

    async def _handle_email_composition(self, message, state):
        if state['step'] == 'subject':
            state['draft']['subject'] = message.content
            state['step'] = 'body'
            return {
                'type': 'quick_reply',
                'text': "✍️ Email body:",
                'options': {'attach': '📎 Add Attachment', 'send': '📤 Send Now'}
            }

        elif state['step'] == 'body':
            if message.content == 'attach':
                state['step'] = 'attachment'
                return "📎 Please send the file you want to attach"

            state['draft']['body'] = message.content
            state['step'] = 'confirm_email'
            return {
                'type': 'quick_reply',
                'text': f"📧 Ready to send?\n\nSubject: {state['draft']['subject']}\n\n{state['draft']['body']}",
                'options': {'confirm': '✅ Send', 'cancel': '❌ cancel'}
            }

        elif state['step'] == 'attachment':
            # Handle attachment upload
            file_type = message.type
            if file_type not in ['document', 'image']:
                return "❌ Unsupported file type"

            media_url = getattr(message, file_type).id
            media_data = self.whc.messenger.download_media(media_url=self.whc.messenger.query_media_url(media_id=media_url), mime_type=media_url.type, file_path=".data/temp")
            state['draft']['attachments'].append(media_data)
            state['step'] = 'body'
            return "📎 Attachment added! Add more or send the email"


    def _parse_email_content(self, email_data):
        """Extract readable content from email payload"""
        parts = email_data.get('payload', {}).get('parts', [])
        body = ""
        for part in parts:
            if part['mimeType'] == 'text/plain':
                body += base64.urlsafe_b64decode(part['body']['data']).decode('utf-8')
        return f"Subject: {email_data.get('subject', '')}\nFrom: {email_data.get('from', '')}\n\n{body}"

    def _build_email_draft(self, draft):
        """Create MIME message from draft data"""
        message = MIMEMultipart()
        message['to'] = draft.get('to', '')
        message['subject'] = draft['subject']
        message.attach(MIMEText(draft['body']))

        for attachment in draft['attachments']:
            part = MIMEBase('application', 'octet-stream')
            part.set_payload(attachment)
            encoders.encode_base64(part)
            part.add_header('Content-Disposition', 'attachment')
            message.attach(part)

        return {'raw': base64.urlsafe_b64encode(message.as_bytes()).decode()}

    def _get_email_subject(self, msg):
        headers = msg.get('payload', {}).get('headers', [])
        return next((h['value'] for h in headers if h['name'] == 'Subject'), 'No Subject')

    def _get_email_sender(self, msg):
        headers = msg.get('payload', {}).get('headers', [])
        return next((h['value'] for h in headers if h['name'] == 'From'), 'Unknown Sender')

    def _get_email_snippet(self, msg):
        return msg.get('snippet', '')[:100] + '...'
    # Calendar Handlers

    # Calendar Functions
    def _format_event_time(self, event):
        """Improved time formatting for calendar events"""
        start = event['start'].get('dateTime', event['start'].get('date'))
        end = event['end'].get('dateTime', event['end'].get('date'))

        try:
            start_dt = parser.parse(start)
            end_dt = parser.parse(end)
            if 'T' in start:
                return f"{start_dt.strftime('%a %d %b %H:%M')} - {end_dt.strftime('%H:%M')}"
            return f"{start_dt.strftime('%d %b %Y')} (All Day)"
        except:
            return "Time not specified"

    async def get_event_details(self, event_id):
        """Retrieve and format calendar event details with location support"""
        if not self.calendar_service:
            return "⚠️ Calendar service not configured"

        try:
            event = self.calendar_service.events().get(
                calendarId='primary',
                eventId=event_id
            ).execute()

            response = [ (
                    f"📅 *Event Details*\n\n"
                    f"Title: {event.get('summary', 'No title')}\n"
                    f"Time: {self._format_event_time(event)}\n"
                    f"Location: {event.get('location', 'Not specified')}\n\n"
                    f"{event.get('description', 'No description')[:1000]}"
                )]

            if 'geo' in event:
                response.append({
                    'lat': float(event['geo']['latitude']),
                    'long': float(event['geo']['longitude']),
                    'name': event.get('location', 'Event Location'),
                    'address': event.get('location', ''),
                    'recipient_id': self.whc.progress_messenger0.recipient_phone
                })
            return response
        except Exception as e:
            return f"⚠️ Error fetching event: {str(e)}"

    async def show_today_events(self, message):
        """Show today's calendar events"""
        if not self.calendar_service:
            message.replay("service not online")

        now = datetime.utcnow().isoformat() + 'Z'
        end_of_day = (datetime.now() + timedelta(days=1)).replace(
            hour=0, minute=0, second=0).isoformat() + 'Z'

        events_result = self.calendar_service.events().list(
            calendarId='primary',
            timeMin=now,
            timeMax=end_of_day,
            singleEvents=True,
            orderBy='startTime'
        ).execute()

        events = events_result.get('items', [])
        return self._format_calendar_response(events, "Today's Events")

    # Updated Calendar List Handlers
    async def show_upcoming_events(self, message):
        """Show upcoming events with interactive support"""
        if not self.calendar_service:
            return "⚠️ Calendar service not configured"

        try:
            now = datetime.utcnow().isoformat() + 'Z'
            next_week = (datetime.now() + timedelta(days=7)).isoformat() + 'Z'

            events_result = self.calendar_service.events().list(
                calendarId='primary',
                timeMin=now,
                timeMax=next_week,
                singleEvents=True,
                orderBy='startTime',
                maxResults=10
            ).execute()

            events = events_result.get('items', [])
            return self._format_calendar_response(events, "Upcoming Events")
        except Exception as e:
            return f"⚠️ Error fetching events: {str(e)}"

    async def start_event_create(self, message):
        """Initiate event creation workflow"""
        self.pending_actions[self.whc.progress_messenger0.recipient_phone] = {
            'type': 'create_event',
            'step': 'title',
            'event_data': {}
        }
        return {
            'type': 'quick_reply',
            'text': "Let's create an event! What's the title?",
            'options': {'cancel': '❌ Cancel'}
        }

    async def find_time_slot(self, message):
        """Find and display the next 5 available time slots with dynamic durations"""
        if not self.calendar_service:
            return "⚠️ Calendar service not configured"

        try:
            # Define the time range for the search (next 24 hours)
            now = datetime.now(UTC)
            end_time = now + timedelta(days=1)

            # FreeBusy Request
            freebusy_request = {
                "timeMin": now.isoformat(),
                "timeMax": end_time.isoformat(),
                "items": [{"id": 'primary'}]
            }

            freebusy_response = self.calendar_service.freebusy().query(body=freebusy_request).execute()
            busy_slots = freebusy_response['calendars']['primary']['busy']

            # Slot-Berechnung
            available_slots = self._calculate_efficient_slots(
                busy_slots,
                self.duration_minutes
            )

            # Format the response for WhatsApp
            return {
                'type': 'list',
                'header': "⏰ Available Time Slots",
                'body': "Tap to select a time slot",
                'footer': "Time Slot Finder",
                'sections': [{
                    'title': "Next 5 Available Slots",
                    'rows': [{
                        'id': f"slot_{slot['start'].timestamp()}",
                        'title': f"🕒 {slot['start'].strftime('%H:%M')} - {slot['end'].strftime('%H:%M')}",
                        'description': f"Duration: {slot['duration']}"
                    } for slot in available_slots[:5]]
                }]
            }
        except Exception as e:
            return f"⚠️ Error finding time slots: {str(e)}"

    def _calculate_efficient_slots(self, busy_slots, duration_minutes):
        """Effiziente Slot-Berechnung"""
        available_slots = []
        current = datetime.now(UTC)
        end_time = current + timedelta(days=1)

        while current < end_time:
            slot_end = current + timedelta(minutes=duration_minutes)

            if slot_end > end_time:
                break

            is_available = all(
                slot_end <= parser.parse(busy['start']) or
                current >= parser.parse(busy['end'])
                for busy in busy_slots
            )

            if is_available:
                available_slots.append({
                    'start': current,
                    'end': slot_end,
                    'duration': f"{duration_minutes} min"
                })
                current = slot_end
            else:
                current += timedelta(minutes=15)

        return available_slots

    async def handle_calendar_actions(self, message):
        """Handle calendar-related pending actions"""
        user_state = self.pending_actions.get(self.whc.progress_messenger0.recipient_phone, {})

        if user_state.get('type') == 'create_event':
            return await self._handle_event_creation(message, user_state)

        return None

    async def _handle_event_creation(self, message, state):
        step = state['step']
        event_data = state['event_data']

        if step == 'title':
            event_data['summary'] = message.content
            state['step'] = 'start_time'
            return "📅 When should it start? (e.g., 'tomorrow 2pm' or '2024-03-20 14:30')"

        elif step == 'start_time':
            event_data['start'] = self._parse_time(message.content)
            state['step'] = 'end_time'
            return "⏰ When should it end? (e.g., '3pm' or '2024-03-20 15:30')"

        elif step == 'end_time':
            event_data['end'] = self._parse_time(message.content, reference=event_data['start'])
            state['step'] = 'description'
            return "📝 Add a description (or type 'skip')"

        elif step == 'description':
            if message.content.lower() != 'skip':
                event_data['description'] = message.content
            state['step'] = 'confirm_envet'
            return self._create_confirmation_message(event_data)

    def _format_calendar_response(self, events, title):
        """Enhanced calendar formatting with interactive support"""
        if not events:
            return f"📅 No {title.lower()} found"

        return {
            'type': 'list',
            'header': title,
            'body': "Tap to view event details",
            "footer": "-- Calendar --",
            'sections': [{
                'title': f"{len(events)} Events",
                'rows': [{
                    'id': f"event_{event['id']}",
                    'title': f"📅 {event['summary']}"[:23],
                    'description': self._format_event_time(event)[:45]
                } for event in events[:5]]
            }]
        }

    def _parse_iso_to_readable(self, iso_str):
        """Convert ISO datetime to readable format"""
        dt = datetime.fromisoformat(iso_str.replace('Z', '+00:00'))
        return dt.strftime("%a %d %b %Y %H:%M")

    def _parse_time(self, time_str, reference=None):
        """
        Konvertiert natürliche Sprache zu präziser Datetime

        Unterstützt:
        - 'heute'
        - 'morgen'
        - 'in einer woche'
        - '10 uhr'
        - '10pm'
        - 'nächsten montag'
        """
        if reference is None:
            reference = datetime.now()

        try:
            import dateparser

            # Dateparser für flexibel Zeitparsing
            parsed_time = dateparser.parse(
                time_str,
                settings={
                    'PREFER_DATES_FROM': 'future',
                    'RELATIVE_BASE': reference,
                    'TIMEZONE': 'Europe/Berlin'
                }
            )

            if parsed_time is None:
                # Fallback auf dateutil wenn dateparser scheitert
                parsed_time = parser .parse(time_str, fuzzy=True, default=reference)

            return parsed_time

        except Exception as e:
            print(f"Zeitparsing-Fehler: {e}")
            return reference

    def _calculate_free_slots(self, start, end, busy_slots):
        """Calculate free time slots between busy periods"""
        # Implementation would calculate available windows
        return [{
            'start': "09:00",
            'end': "11:00",
            'duration': "2 hours"
        }]

    def _create_confirmation_message(self, event_data):
        """Create event confirmation message"""
        details = [
            f"📌 Title: {event_data['summary']}",
            f"🕒 Start: {self._parse_iso_to_readable(event_data['start'])}",
            f"⏰ End: {self._parse_iso_to_readable(event_data['end'])}",
            f"📝 Description: {event_data.get('description', 'None')}"
        ]
        return {
            'type': 'quick_reply',
            'text': "\n".join(details),
            'options': {'confirm': '✅ Confirm', 'cancel': '❌ Cancel'}
        }

    def _create_calendar_event(self, event_data):
        """Create event through Calendar API"""
        event = {
            'summary': event_data['summary'],
            'start': {'dateTime': event_data['start']},
            'end': {'dateTime': event_data['end']},
        }
        if 'description' in event_data:
            event['description'] = event_data['description']

        return self.calendar_service.events().insert(
            calendarId='primary',
            body=event
        ).execute()

    async def system_status(self, message):
        o = (datetime.now() - self.start_time)
        o.microseconds = 0
        status = {
            "🤖 Agent": "Online" if self.agent else "Offline",
            "📧 Email": "Connected" if self.gmail_service else "Disconnected",
            "📅 Calendar": "Connected" if self.calendar_service else "Disconnected",
            "📄 Documents": "Connected" if self.blob_docs_system else "Disconnected",
            "⏳ Uptime": f"{str(o.isoformat())}"
        }
        return "\n".join([f"{k}: {v}" for k, v in status.items()])

    async def restart_system(self, message):
        message.reply("🔄 System restart initiated...")
        time.sleep(1)
        await self.clear_memory(message)
        time.sleep(1)
        return  "✅ System restarted"

    # Updated document handlers
    async def list_documents(self, message, filter_type=None):
        docs = self.blob_docs_system.list_documents(filter_type)
        if len(docs) == 0:
            return "No docs found"
        else:
            return str(docs)
        return {
            'type': 'list',
            'body': 'Stored Documents',
            'action': {
                'sections': [{
                    'title': 'Your Documents',
                    'rows': [{
                        'id': doc['id'],
                        'title': f"{self._get_icon(doc['type'])} {doc['name']}"[:23],
                        'description': f"{doc['type'].title()} | {self._format_size(doc['size'])} | {doc['modified']}"[:29]
                    } for doc in docs[:10]]
                }]}
        }

    async def start_document_upload(self, message):
        """Initiate document upload workflow"""
        self.pending_actions[self.whc.progress_messenger0.recipient_phone] = {'type': 'document', 'step': 'awaiting_file'}
        return {
            'type': 'quick_reply',
            'text': '📤 Send me the file you want to upload',
            'options': {'cancel': '❌ Cancel Upload'}
        }

    async def search_documents(self, message):
        """Initiate document search workflow"""
        self.pending_actions[self.whc.progress_messenger0.recipient_phone] = {'type': 'search', 'step': 'awaiting_query'}
        return {
            'type': 'quick_reply',
            'text': '🔍 What are you looking for?',
            'options': {'cancel': '❌ Cancel Search'}
        }

    async def handle_media_message(self, message: 'Message'):
        """Handle document/image/video uploads"""
        user_state = self.pending_actions.get(self.whc.progress_messenger0.recipient_phone, {})

        if user_state.get('step') == 'awaiting_file':
            file_type = message.type
            if file_type not in ['document', 'image', 'video']:
                return "Unsupported file type"

            try:
                # Download media
                #media_url = message.document.url if hasattr(message, 'document') else \
                #    message.image.url if hasattr(message, 'image') else \
                #        message.video.url
                if file_type =='video':
                    content = self.whc.messenger.get_video(message.data)
                if file_type =='image':
                    content = self.whc.messenger.get_image(message.data)
                if file_type =='document':
                    content = self.whc.messenger.get_document(message.data)
                print("Media content:", content)
                media_data = self.whc.messenger.download_media(media_url=self.whc.messenger.query_media_url(media_id=content.get('id')),  mime_type=content.get('mime_type'), file_path='.data/temp')
                print("Media media_data:", media_data)
                # Save to blob storage
                filename = f"file_{file_type}_{datetime.now().isoformat()}_{content.get('sha256', '')}"
                blob_id = self.blob_docs_system.save_document(
                    open(media_data, 'rb').read(),
                    filename=filename,
                    file_type=file_type
                )

                self.pending_actions[self.whc.progress_messenger0.recipient_phone] = {}
                return f"✅ File uploaded successfully!\nID: {blob_id}"

            except Exception as e:
                logging.error(f"Upload failed: {str(e)}")
                return f"❌ Failed to upload file Error : {str(e)}"

        return "No pending uploads"

    async def delete_document(self, message):
        """Delete document workflow"""
        docs = self.blob_docs_system.list_documents()
        return {
            'type': 'quick_reply',
            'text': 'Select document to delete:',
            'options': {doc['id']: doc['name'] for doc in docs[:5]},
            'handler': self._confirm_delete
        }

    async def _confirm_delete(self, doc_id, message):
        """Confirm deletion workflow"""
        doc = next((d for d in self.blob_docs_system.list_documents() if d['id'] == doc_id), None)
        if not doc:
            return "Document not found"

        if self.blob_docs_system.delete_document(doc_id):
            return f"✅ {doc['name']} deleted successfully"
        return "❌ Failed to delete document"

    # Helper methods
    def _get_icon(self, file_type: str) -> str:
        icons = {
            'document': '📄',
            'image': '🖼️',
            'video': '🎥'
        }
        return icons.get(file_type, '📁')

    def _format_size(self, size: int) -> str:
        if size < 1024:
            return f"{size}B"
        elif size < 1024 ** 2:
            return f"{size / 1024:.1f}KB"
        elif size < 1024 ** 3:
            return f"{size / (1024 ** 2):.1f}MB"
        return f"{size / (1024 ** 3):.1f}GB"

    # Utility Methods

    def _clean_processed_messages(self):
        """Clean old messages from processed cache"""
        now = time.time()
        self.processed_messages = {
            msg_id for msg_id, timestamp in self.processed_messages
            if now - timestamp < 3600  # 1 hour retention
        }

    def send_email(self, to, subject, body):
        """Actual email sending function to be called by agent"""
        if not self.gmail_service:
            return False

        message = MIMEText(body)
        message['to'] = to
        message['subject'] = subject

        encoded_message = base64.urlsafe_b64encode(message.as_bytes()).decode()
        self.gmail_service.users().messages().send(
            userId='me',
            body={'raw': encoded_message}
        ).execute()
        return True

    async def start_agent(self, *a):
        """Start the agent in background mode"""
        if self.agent:
            self.agent.run_in_background()
            return True
        return False

    async def stop_agent(self, *b):
        """Stop the currently running agent"""
        if self.agent:
            self.agent.stop()
            return True
        return False

    async def show_task_stack(self, *a):
        """Display current task stack"""
        if self.agent and len(self.agent.taskstack.tasks) > 0:
            tasks = self.agent.taskstack.tasks
            return self.agent.mini_task("\n".join([f"Task {t.id}: {t.description}" for t in tasks]), "system", "Format to nice and clean whatsapp format")
        return "No tasks in stack"

    def run(self):
        """Start the WhatsApp assistant"""
        try:
            self.state = AssistantState.ONLINE
            # Send welcome message

            mas = self.whc.messenger.create_message(
                content="Digital Assistant is online! Send /help for available commands.",to=self.whc.progress_messenger0.recipient_phone,
            ).send(sender=0)
            mas_id = mas.get("messages", [{}])[0].get("id")
            print(mas_id)

        except Exception as e:
            logging.error(f"Assistant error: {str(e)}")
            self.state = AssistantState.OFFLINE
            raise

    async def handle_agent_actions(self, message):
        user_state = self.pending_actions.get(self.whc.progress_messenger0.recipient_phone, {})
        def helper():

            stop_flag = threading.Event()
            try:
                progress = self.progress_messengers['task']
                # message_id = progress.send_initial_message(mode="loading")
                progress.message_id = message.id
                progress.start_loading_in_background(stop_flag)
                res = message.content
                print(message.data.get('entry', [{}])[0].get('changes', [{}])[0].get('value', {}).get('messages', [{}])[0].get(
                    'context'))
                if context := message.data.get('entry', [{}])[0].get('changes', [{}])[0].get('value', {}).get('messages', [{}])[0].get(
                    'context'):
                    context_str = f"Context : source {'USER' if context.get('from') in self.whc.progress_messenger0.recipient_phone else 'AGENT'}"
                    cd = self.history.get(context.get('id'))
                    context_str += "\n" + (cd if cd is not None else "The ref Message is not in the history")
                    res += "\n" + context_str
                if user_state.get('type') == 'system':
                    res = self.isaa.run(res)
                    self.pending_actions[self.whc.progress_messenger0.recipient_phone] = {}
                elif user_state.get('type') == 'self-agent':
                    res = self.agent.run(res)
                    self.pending_actions[self.whc.progress_messenger0.recipient_phone] = {}
                self.agent.mode = LLMMode(
                    name="Chatter",
                    description="whatsapp Chat LLM",
                    system_msg="Response precise and short style using whatsapp syntax!",
                    post_msg=None
                )
                response = self.agent.mini_task(res, "user", persist=True)
                self.save_reply(message, response)
            except Exception as e:
                stop_flag.set()
                message.reply("❌ Error in agent "+str(e))
            finally:
                self.agent.mode = None
                stop_flag.set()
        threading.Thread(target=helper, daemon=True).start()

    def save_reply(self, message, content):
        res = message.reply(content)
        res_id = res.get("messages", [{}])[0].get("id")
        if res_id is not None:
            self.history.set(res_id, content)
        else:
            print(f"No ID to add to history: {res}")

agent_task(message) async

Initiate email search workflow

Source code in toolboxv2/mods/WhatsAppTb/client.py

async def agent_task(self, message):
    """Initiate email search workflow"""
    self.pending_actions[self.whc.progress_messenger0.recipient_phone] = {
        'type': 'self-agent',
        'step': 'await_query'
    }
    return {
        'type': 'quick_reply',
        'text': "Now prompt the self-agent 📝",
        'options': {'cancel': '❌ Cancel Search'}
    }

check_emails(message, query='') async

Improved email checking with WhatsApp API formatting

Source code in toolboxv2/mods/WhatsAppTb/client.py

async def check_emails(self, message, query=""):
    """Improved email checking with WhatsApp API formatting"""
    if not self.gmail_service:
        return "⚠️ Gmail service not configured"

    try:
        results = self.gmail_service.users().messages().list(
            userId='me',
            maxResults=10,
            labelIds=['INBOX'],
            q=query
        ).execute()

        emails = []
        for msg in results.get('messages', [])[:10]:
            email_data = self.gmail_service.users().messages().get(
                userId='me',
                id=msg['id'],
                format='metadata'
            ).execute()

            headers = {h['name']: h['value'] for h in email_data['payload']['headers']}
            emails.append({
                'id': msg['id'],
                'from': headers.get('From', 'Unknown'),
                'subject': headers.get('Subject', 'No Subject'),
                'date': headers.get('Date', 'Unknown'),
                'snippet': email_data.get('snippet', ''),
                'unread': 'UNREAD' in email_data.get('labelIds', [])
            })

        return {
            'type': 'list',
            'header': '📨 Recent Emails',
            'body': 'Tap to view full email',
            'footer': 'Email Manager',
            'sections': [{
                'title': f"Inbox ({len(emails)} emails)",
                'rows': [{
                    'id': f"email_{email['id']}",
                    'title': f"{'📬' if email['unread'] else '📭'} {email['subject']}"[:23],
                    'description': f"From: {email['from']}\n{email['snippet']}"[:45]
                } for email in emails]
            }]
        }
    except Exception as e:
        return f"⚠️ Error fetching emails: {str(e)}"

complete_authorization(message)

Complete the authorization process using the authorization code

:param authorization_code: Authorization code received from Google

Source code in toolboxv2/mods/WhatsAppTb/client.py

def complete_authorization(self, message: Message):
    """
    Complete the authorization process using the authorization code

    :param authorization_code: Authorization code received from Google
    """
    from google_auth_oauthlib.flow import Flow
    authorization_code = message.content
    # Define the scopes required for Gmail and Calendar
    SCOPES = [
        'https://www.googleapis.com/auth/gmail.modify',
        'https://www.googleapis.com/auth/calendar'
    ]

    # Create a flow instance to manage the OAuth 2.0 authorization process
    flow = Flow.from_client_secrets_file(
        self.credentials_path,
        scopes=SCOPES,
        redirect_uri='urn:ietf:wg:oauth:2.0:oob'
    )

    # Exchange the authorization code for credentials
    flow.fetch_token(code=authorization_code)
    self.credentials = flow.credentials

    # Save the credentials for future use
    self.save_credentials()

    # Initialize services
    self.init_services()
    return "Done"

delete_document(message) async

Delete document workflow

Source code in toolboxv2/mods/WhatsAppTb/client.py

async def delete_document(self, message):
    """Delete document workflow"""
    docs = self.blob_docs_system.list_documents()
    return {
        'type': 'quick_reply',
        'text': 'Select document to delete:',
        'options': {doc['id']: doc['name'] for doc in docs[:5]},
        'handler': self._confirm_delete
    }

email_search(message) async

Initiate email search workflow

Source code in toolboxv2/mods/WhatsAppTb/client.py

async def email_search(self, message):
    """Initiate email search workflow"""
    self.pending_actions[self.whc.progress_messenger0.recipient_phone] = {
        'type': 'email_search',
        'step': 'await_query'
    }
    return {
        'type': 'quick_reply',
        'text': "🔍 What would you like to search for?",
        'options': {'cancel': '❌ Cancel Search'}
    }

email_summary(message) async

Generate AI-powered email summaries

Source code in toolboxv2/mods/WhatsAppTb/client.py

async def email_summary(self, message):
    """Generate AI-powered email summaries"""
    try:
        messages = self.gmail_service.users().messages().list(
            userId='me',
            maxResults=3,
            labelIds=['INBOX']
        ).execute().get('messages', [])

        email_contents = []
        for msg in messages[:3]:
            email_data = self.gmail_service.users().messages().get(
                userId='me',
                id=msg['id'],
                format='full'
            ).execute()
            email_contents.append(self._parse_email_content(email_data))

        summary = self.agent.mini_task(
            "\n\n".join(email_contents) , "system", "Summarize these emails in bullet points with key details:"
        )

        return f"📋 Email Summary:\n{summary}\n\n*Powered by AI*"
    except Exception as e:
        logging.error(f"Summary failed: {str(e)}")
        return f"❌ Could not generate summary: {str(e)}"

find_time_slot(message) async

Find and display the next 5 available time slots with dynamic durations

Source code in toolboxv2/mods/WhatsAppTb/client.py

async def find_time_slot(self, message):
    """Find and display the next 5 available time slots with dynamic durations"""
    if not self.calendar_service:
        return "⚠️ Calendar service not configured"

    try:
        # Define the time range for the search (next 24 hours)
        now = datetime.now(UTC)
        end_time = now + timedelta(days=1)

        # FreeBusy Request
        freebusy_request = {
            "timeMin": now.isoformat(),
            "timeMax": end_time.isoformat(),
            "items": [{"id": 'primary'}]
        }

        freebusy_response = self.calendar_service.freebusy().query(body=freebusy_request).execute()
        busy_slots = freebusy_response['calendars']['primary']['busy']

        # Slot-Berechnung
        available_slots = self._calculate_efficient_slots(
            busy_slots,
            self.duration_minutes
        )

        # Format the response for WhatsApp
        return {
            'type': 'list',
            'header': "⏰ Available Time Slots",
            'body': "Tap to select a time slot",
            'footer': "Time Slot Finder",
            'sections': [{
                'title': "Next 5 Available Slots",
                'rows': [{
                    'id': f"slot_{slot['start'].timestamp()}",
                    'title': f"🕒 {slot['start'].strftime('%H:%M')} - {slot['end'].strftime('%H:%M')}",
                    'description': f"Duration: {slot['duration']}"
                } for slot in available_slots[:5]]
            }]
        }
    except Exception as e:
        return f"⚠️ Error finding time slots: {str(e)}"

generate_authorization_url(*a) async

Generate an authorization URL for user consent

:return: Authorization URL for the user to click and authorize access

Source code in toolboxv2/mods/WhatsAppTb/client.py

async def generate_authorization_url(self, *a):
    """
    Generate an authorization URL for user consent

    :return: Authorization URL for the user to click and authorize access
    """
    from google_auth_oauthlib.flow import Flow
    # Define the scopes required for Gmail and Calendar
    SCOPES = [
        'https://www.googleapis.com/auth/gmail.modify',
        'https://www.googleapis.com/auth/calendar'
    ]

    # Create a flow instance to manage the OAuth 2.0 authorization process
    flow = Flow.from_client_secrets_file(
        self.credentials_path,
        scopes=SCOPES,
        redirect_uri='urn:ietf:wg:oauth:2.0:oob'  # Use 'urn:ietf:wg:oauth:2.0:oob' for desktop apps
    )

    # Generate the authorization URL
    authorization_url, _ = flow.authorization_url(
        access_type='offline',  # Allows obtaining refresh token
        prompt='consent'  # Ensures user is always prompted for consent
    )
    self.pending_actions[self.whc.progress_messenger0.recipient_phone] = {'type': 'auth',
                                                                          'step': 'awaiting_key'}
    return {
        'type': 'quick_reply',
        'text': f'Url to log in {authorization_url}',
        'options': {'cancel': '❌ Cancel Upload'}
    }

get_email_details(email_id) async

Retrieve and format full email details

Source code in toolboxv2/mods/WhatsAppTb/client.py

async def get_email_details(self, email_id):
    """Retrieve and format full email details"""
    if not self.gmail_service:
        return "⚠️ Gmail service not configured"

    try:
        email_data = self.gmail_service.users().messages().get(
            userId='me',
            id=email_id,
            format='full'
        ).execute()

        headers = {h['name']: h['value'] for h in email_data['payload']['headers']}
        body = ""
        for part in email_data.get('payload', {}).get('parts', []):
            if part['mimeType'] == 'text/plain':
                body = base64.urlsafe_b64decode(part['body']['data']).decode('utf-8')
                break

        formatted_text = (
            f"📧 *Email Details*\n\n"
            f"From: {headers.get('From', 'Unknown')}\n"
            f"Subject: {headers.get('Subject', 'No Subject')}\n"
            f"Date: {headers.get('Date', 'Unknown')}\n\n"
            f"{body[:15000]}{'...' if len(body) > 15000 else ''}"
        )
        return  self.agent.mini_task(
            formatted_text , "system", "Summarize the email in bullet points with key details"
        )
    except Exception as e:
        return f"⚠️ Error fetching email: {str(e)}"

get_event_details(event_id) async

Retrieve and format calendar event details with location support

Source code in toolboxv2/mods/WhatsAppTb/client.py

async def get_event_details(self, event_id):
    """Retrieve and format calendar event details with location support"""
    if not self.calendar_service:
        return "⚠️ Calendar service not configured"

    try:
        event = self.calendar_service.events().get(
            calendarId='primary',
            eventId=event_id
        ).execute()

        response = [ (
                f"📅 *Event Details*\n\n"
                f"Title: {event.get('summary', 'No title')}\n"
                f"Time: {self._format_event_time(event)}\n"
                f"Location: {event.get('location', 'Not specified')}\n\n"
                f"{event.get('description', 'No description')[:1000]}"
            )]

        if 'geo' in event:
            response.append({
                'lat': float(event['geo']['latitude']),
                'long': float(event['geo']['longitude']),
                'name': event.get('location', 'Event Location'),
                'address': event.get('location', ''),
                'recipient_id': self.whc.progress_messenger0.recipient_phone
            })
        return response
    except Exception as e:
        return f"⚠️ Error fetching event: {str(e)}"

handle_audio_message(message) async

Process audio messages with STT and TTS

Source code in toolboxv2/mods/WhatsAppTb/client.py

async def handle_audio_message(self, message: 'Message'):
    """Process audio messages with STT and TTS"""
    # Download audio
    progress = self.progress_messengers['task']
    stop_flag = threading.Event()
    # message_id = progress.send_initial_message(mode="loading")
    progress.message_id = message.id
    progress.start_loading_in_background(stop_flag)

    content = self.whc.messenger.get_audio(message.data)
    audio_file_name = self.whc.messenger.download_media(media_url=self.whc.messenger.query_media_url(media_id=content.get('id')), mime_type='audio/opus', file_path=".data/temp")
    print(f"audio_file_name {audio_file_name}")
    if audio_file_name is None:
        message.reply("Could not process audio file")
        stop_flag.set()
        return

    text = self.stt(audio_file_name)['text']
    if not text:
        message.reply("Could not process audio")
        stop_flag.set()
        return

    message.reply("Transcription :\n "+ text)
    message.content = text
    agent_res = await self.helper_text(message, return_text=True)

    if agent_res is not None:
        pass

    stop_flag.set()

handle_button_interaction(content, message) async

Handle button click interactions

Source code in toolboxv2/mods/WhatsAppTb/client.py

async def handle_button_interaction(self, content: dict, message: Message):
    """Handle button click interactions"""
    button_id = content['id']

    # First check if it's a main menu button
    if button_id in self.buttons:
        self.whc.messenger.send_button(
            recipient_id=self.whc.progress_messenger0.recipient_phone,
            button=self.buttons[button_id]
        )
        return

    # Handle action buttons
    action_handlers = {
        # Agent controls
        'start': self.start_agent,
        'stop': self.stop_agent,
        'tasks': self.show_task_stack,
        'memory': self.clear_memory,
        'system-task': self.system_task,
        'agent-task': self.agent_task,

        # Email controls
        'check': self.check_emails,
        'send': self.start_email_compose,
        'summary': self.email_summary,
        'search': self.email_search,

        # Calendar controls
        'today': self.show_today_events,
        'add': self.start_event_create,
        'upcoming': self.show_upcoming_events,
        'find_slot': self.find_time_slot,

        # Document controls
        'upload': self.start_document_upload,
        'list': self.list_documents,
        'search_docs': self.search_documents,
        'delete': self.delete_document,

        # System controls
        'status': self.system_status,
        'restart': self.restart_system,
        'connect': self.generate_authorization_url,

        'cancel': self.cancel,
        'confirm': self.confirm,
    }
    if button_id in action_handlers:
        try:
            # Start progress indicator
            progress = self.progress_messengers['task']
            stop_flag = threading.Event()
            # message_id = progress.send_initial_message(mode="loading")
            progress.message_id = message.id
            progress.start_loading_in_background(stop_flag)

            # Execute handler

            result = await action_handlers[button_id](message)


            # Send result
            if isinstance(result, str):
                self.save_reply(message, result)
            elif isinstance(result, dict):  # For structured responses
                self.send_structured_response(result)

            stop_flag.set()
        finally:
            #except Exception as e:
            stop_flag.set()
        #    message.reply(f"❌ Error processing {button_id}: {str(e)}")
    elif 'event_' in button_id:
        res = await self.get_event_details(button_id.replace("event_", ''))
        if isinstance(res, str):
            self.save_reply(message, res)
            return
        for r in res:
            if isinstance(r, str):
                self.save_reply(message, r)
            else:
                self.whc.messenger.send_location(**r)

    elif 'email_' in button_id:
        res = await self.get_email_details(button_id.replace("email_", ''))
        self.save_reply(message, res)
    else:
        message.reply("⚠️ Unknown command")

handle_calendar_actions(message) async

Handle calendar-related pending actions

Source code in toolboxv2/mods/WhatsAppTb/client.py

async def handle_calendar_actions(self, message):
    """Handle calendar-related pending actions"""
    user_state = self.pending_actions.get(self.whc.progress_messenger0.recipient_phone, {})

    if user_state.get('type') == 'create_event':
        return await self._handle_event_creation(message, user_state)

    return None

handle_email_actions(message) async

Handle multi-step email workflows

Source code in toolboxv2/mods/WhatsAppTb/client.py

    async def handle_email_actions(self, message):
        """Handle multi-step email workflows"""
        user_state = self.pending_actions.get(self.whc.progress_messenger0.recipient_phone, {})

        if user_state.get('type') == 'compose_email':
            return await self._handle_email_composition(message, user_state)
        if user_state.get('type') == 'email_search':
            return await self.check_emails(message, self.agent.mini_task("""Conventire Pezise zu einer googel str only query using : Gmail Suchoperatoren!

Basis-Operatoren:
- from: Absender
- to: Empfänger
- subject: Betreff
- label: Gmail Label
- has:attachment Anhänge
- newer_than:7d Zeitfilter
- before: Datum vor
- after: Datum nach

Erweiterte Operatoren:
- in:inbox
- in:sent
- in:spam
- cc: Kopie
- bcc: Blindkopie
- is:unread
- is:read
- larger:10M Größenfilter
- smaller:5M
- filename:pdf Dateityp

Profi-Tipps:
- Kombinierbar mit UND/ODER
- Anführungszeichen für exakte Suche
- Negation mit -
 beispeile : 'Ungelesene Mails letzte Woche': -> 'is:unread newer_than:7d'

""", "user",message.content))


        return None

handle_interactive(message) async

Handle all interactive messages

Source code in toolboxv2/mods/WhatsAppTb/client.py

async def handle_interactive(self, message: Message):
    """Handle all interactive messages"""
    content = self.whc.messenger.get_interactive_response(message.data)
    if content.get("type") == "list_reply":
        await self.handle_button_interaction(content.get("list_reply"), message)
    elif content.get("type") == "button_reply":
        print(content)

handle_media_message(message) async

Handle document/image/video uploads

Source code in toolboxv2/mods/WhatsAppTb/client.py

async def handle_media_message(self, message: 'Message'):
    """Handle document/image/video uploads"""
    user_state = self.pending_actions.get(self.whc.progress_messenger0.recipient_phone, {})

    if user_state.get('step') == 'awaiting_file':
        file_type = message.type
        if file_type not in ['document', 'image', 'video']:
            return "Unsupported file type"

        try:
            # Download media
            #media_url = message.document.url if hasattr(message, 'document') else \
            #    message.image.url if hasattr(message, 'image') else \
            #        message.video.url
            if file_type =='video':
                content = self.whc.messenger.get_video(message.data)
            if file_type =='image':
                content = self.whc.messenger.get_image(message.data)
            if file_type =='document':
                content = self.whc.messenger.get_document(message.data)
            print("Media content:", content)
            media_data = self.whc.messenger.download_media(media_url=self.whc.messenger.query_media_url(media_id=content.get('id')),  mime_type=content.get('mime_type'), file_path='.data/temp')
            print("Media media_data:", media_data)
            # Save to blob storage
            filename = f"file_{file_type}_{datetime.now().isoformat()}_{content.get('sha256', '')}"
            blob_id = self.blob_docs_system.save_document(
                open(media_data, 'rb').read(),
                filename=filename,
                file_type=file_type
            )

            self.pending_actions[self.whc.progress_messenger0.recipient_phone] = {}
            return f"✅ File uploaded successfully!\nID: {blob_id}"

        except Exception as e:
            logging.error(f"Upload failed: {str(e)}")
            return f"❌ Failed to upload file Error : {str(e)}"

    return "No pending uploads"

handle_message(message) async

Main message handler for incoming WhatsApp messages

Source code in toolboxv2/mods/WhatsAppTb/client.py

async def handle_message(self, message: 'Message'):
    """Main message handler for incoming WhatsApp messages"""

    # Deduplication check
    with self.message_lock:
        if message.id in self.processed_messages:
            return
        last_ts = time.time()
        print(last_ts)
        if len(self.processed_messages) > 0:
            m_id, last_ts = self.processed_messages.pop()
            self.processed_messages.add((m_id, last_ts))

        print("DUPLICATION P", message.data.get('entry', [{}])[0].get('changes', [{}])[0].get('value', {}).get('messages', [{}])[0].get('timestamp', 0) , last_ts)
        if float(message.data.get('entry', [{}])[0].get('changes', [{}])[0].get('value', {}).get('messages', [{}])[0].get('timestamp', 0)) < last_ts - 120:
            return
        self.processed_messages.add((message.id, time.perf_counter()))

    # Mark message as read
    message.mark_as_read()

    # Extract content and type
    content_type = message.type
    content = message.content

    print(f"message.content {content=} {content_type=} {message.data=}")

    try:
        if content_type == 'interactive':
            await self.handle_interactive(message)
        elif content_type == 'audio':
            await self.handle_audio_message(message)
        elif content_type in ['document', 'image', 'video']:
            response = await self.handle_media_message(message)
            self.save_reply(message, response)
        elif content_type == 'text':
            if content.lower() == "menu":
                self.whc.messenger.send_button(
                    recipient_id=self.whc.progress_messenger0.recipient_phone,
                    button=self.buttons[content.lower()]
                )
            else:
                await self.helper_text(message)
        else:
            message.reply("Unsupported message type")
    #except Exception as e:
    #    logging.error(f"Message handling error: {str(e)}")
    #   message.reply("❌ Error processing request")
    finally:
        # Cleanup old messages (keep 1 hour history)
        with self.message_lock:
            self._clean_processed_messages()

init_services()

Initialize Gmail and Calendar services

Source code in toolboxv2/mods/WhatsAppTb/client.py

def init_services(self):
    """
    Initialize Gmail and Calendar services
    """
    from googleapiclient.discovery import build

    self.gmail_service = build('gmail', 'v1', credentials=self.credentials)
    self.calendar_service = build('calendar', 'v3', credentials=self.credentials)
    self.pending_actions[self.whc.progress_messenger0.recipient_phone] = {}

load_credentials()

Load previously saved credentials if available

:return: Whether credentials were successfully loaded

Source code in toolboxv2/mods/WhatsAppTb/client.py

def load_credentials(self):
    """
    Load previously saved credentials if available

    :return: Whether credentials were successfully loaded
    """
    try:
        self.credentials = Credentials.from_authorized_user_file('token/google_token.json')
        self.init_services()
        return True
    except FileNotFoundError:
        return False

run()

Start the WhatsApp assistant

Source code in toolboxv2/mods/WhatsAppTb/client.py

def run(self):
    """Start the WhatsApp assistant"""
    try:
        self.state = AssistantState.ONLINE
        # Send welcome message

        mas = self.whc.messenger.create_message(
            content="Digital Assistant is online! Send /help for available commands.",to=self.whc.progress_messenger0.recipient_phone,
        ).send(sender=0)
        mas_id = mas.get("messages", [{}])[0].get("id")
        print(mas_id)

    except Exception as e:
        logging.error(f"Assistant error: {str(e)}")
        self.state = AssistantState.OFFLINE
        raise

save_credentials()

Save the obtained credentials to a file for future use

Source code in toolboxv2/mods/WhatsAppTb/client.py

def save_credentials(self):
    """
    Save the obtained credentials to a file for future use
    """
    if not os.path.exists('token'):
        os.makedirs('token')

    with open('token/google_token.json', 'w') as token_file:
        token_file.write(self.credentials.to_json())

search_documents(message) async

Initiate document search workflow

Source code in toolboxv2/mods/WhatsAppTb/client.py

async def search_documents(self, message):
    """Initiate document search workflow"""
    self.pending_actions[self.whc.progress_messenger0.recipient_phone] = {'type': 'search', 'step': 'awaiting_query'}
    return {
        'type': 'quick_reply',
        'text': '🔍 What are you looking for?',
        'options': {'cancel': '❌ Cancel Search'}
    }

send_email(to, subject, body)

Actual email sending function to be called by agent

Source code in toolboxv2/mods/WhatsAppTb/client.py

def send_email(self, to, subject, body):
    """Actual email sending function to be called by agent"""
    if not self.gmail_service:
        return False

    message = MIMEText(body)
    message['to'] = to
    message['subject'] = subject

    encoded_message = base64.urlsafe_b64encode(message.as_bytes()).decode()
    self.gmail_service.users().messages().send(
        userId='me',
        body={'raw': encoded_message}
    ).execute()
    return True

send_structured_response(result)

Send complex responses using appropriate WhatsApp features

Source code in toolboxv2/mods/WhatsAppTb/client.py

def send_structured_response(self, result: dict):
    """Send complex responses using appropriate WhatsApp features"""
    if result['type'] == 'list':
        self.whc.messenger.send_button(
            recipient_id=self.whc.progress_messenger0.recipient_phone,
            button={
                'header': result.get('header', ''),
                'body': result.get('body', ''),
                'footer': result.get('footer', ''),
                'action': {
                    'button': 'Action',
                    'sections': result['sections']
                }
            }
        )
    elif result['type'] == 'quick_reply':
        self.whc.messenger.send_button(
            recipient_id=self.whc.progress_messenger0.recipient_phone,
            button={
                'header': "Quick reply",
                'body': result['text'],
                'footer': '',
                'action': {'button': 'Action', 'sections': [{
                    'title': 'View',
                    'rows': [{'id': k, 'title': v[:23]} for k, v in result['options'].items()]
                }]}
            }
        )

    elif result['type'] == 'media':
        if result['media_type'] == 'image':
            self.whc.messenger.send_image(
                image=result['url'],
                recipient_id=self.whc.progress_messenger0.recipient_phone,
                caption=result.get('caption', '')
            )
        elif result['media_type'] == 'document':
            self.whc.messenger.send_document(
                document=result['url'],
                recipient_id=self.whc.progress_messenger0.recipient_phone,
                caption=result.get('caption', '')
            )

setup_interaction_buttons()

Define WhatsApp interaction buttons for different functionalities

Source code in toolboxv2/mods/WhatsAppTb/client.py

def setup_interaction_buttons(self):
    """Define WhatsApp interaction buttons for different functionalities"""
    self.buttons = {
        'menu': {
            'header': 'Digital Assistant',
            'body': 'Please select an option:',
            'footer': '-- + --',
            'action': {
                'button': 'Menu',
                'sections': [
                    {
                        'title': 'Main Functions',
                        'rows': [
                            {'id': 'agent', 'title': 'Agent Controls', 'description': 'Manage your AI assistant'},
                            {'id': 'email', 'title': 'Email Management', 'description': 'Handle your emails'},
                            {'id': 'calendar', 'title': 'Calendar', 'description': 'Manage your schedule'},
                            {'id': 'docs', 'title': 'Documents', 'description': 'Handle documents'},
                            {'id': 'system', 'title': 'System', 'description': 'System controls and metrics'}
                        ]
                    }
                ]
            }
        },
        'agent': self._create_agent_controls_buttons(),
        'email': self._create_email_controls_buttons(),
        'calendar': self._create_calendar_controls_buttons(),
        'docs': self._create_docs_controls_buttons(),
        'system': self._create_system_controls_buttons()
    }

setup_progress_messengers()

Initialize progress messengers for different types of tasks

Source code in toolboxv2/mods/WhatsAppTb/client.py

def setup_progress_messengers(self):
    """Initialize progress messengers for different types of tasks"""
    self.progress_messengers = {
        'task': self.whc.progress_messenger0,
        'email': self.whc.progress_messenger1,
        'calendar': self.whc.progress_messenger2
    }

show_task_stack(*a) async

Display current task stack

Source code in toolboxv2/mods/WhatsAppTb/client.py

async def show_task_stack(self, *a):
    """Display current task stack"""
    if self.agent and len(self.agent.taskstack.tasks) > 0:
        tasks = self.agent.taskstack.tasks
        return self.agent.mini_task("\n".join([f"Task {t.id}: {t.description}" for t in tasks]), "system", "Format to nice and clean whatsapp format")
    return "No tasks in stack"

show_today_events(message) async

Show today's calendar events

Source code in toolboxv2/mods/WhatsAppTb/client.py

async def show_today_events(self, message):
    """Show today's calendar events"""
    if not self.calendar_service:
        message.replay("service not online")

    now = datetime.utcnow().isoformat() + 'Z'
    end_of_day = (datetime.now() + timedelta(days=1)).replace(
        hour=0, minute=0, second=0).isoformat() + 'Z'

    events_result = self.calendar_service.events().list(
        calendarId='primary',
        timeMin=now,
        timeMax=end_of_day,
        singleEvents=True,
        orderBy='startTime'
    ).execute()

    events = events_result.get('items', [])
    return self._format_calendar_response(events, "Today's Events")

show_upcoming_events(message) async

Show upcoming events with interactive support

Source code in toolboxv2/mods/WhatsAppTb/client.py

async def show_upcoming_events(self, message):
    """Show upcoming events with interactive support"""
    if not self.calendar_service:
        return "⚠️ Calendar service not configured"

    try:
        now = datetime.utcnow().isoformat() + 'Z'
        next_week = (datetime.now() + timedelta(days=7)).isoformat() + 'Z'

        events_result = self.calendar_service.events().list(
            calendarId='primary',
            timeMin=now,
            timeMax=next_week,
            singleEvents=True,
            orderBy='startTime',
            maxResults=10
        ).execute()

        events = events_result.get('items', [])
        return self._format_calendar_response(events, "Upcoming Events")
    except Exception as e:
        return f"⚠️ Error fetching events: {str(e)}"

start_agent(*a) async

Start the agent in background mode

Source code in toolboxv2/mods/WhatsAppTb/client.py

async def start_agent(self, *a):
    """Start the agent in background mode"""
    if self.agent:
        self.agent.run_in_background()
        return True
    return False

start_document_upload(message) async

Initiate document upload workflow

Source code in toolboxv2/mods/WhatsAppTb/client.py

async def start_document_upload(self, message):
    """Initiate document upload workflow"""
    self.pending_actions[self.whc.progress_messenger0.recipient_phone] = {'type': 'document', 'step': 'awaiting_file'}
    return {
        'type': 'quick_reply',
        'text': '📤 Send me the file you want to upload',
        'options': {'cancel': '❌ Cancel Upload'}
    }

start_email_compose(message) async

Enhanced email composition workflow

Source code in toolboxv2/mods/WhatsAppTb/client.py

async def start_email_compose(self, message):
    """Enhanced email composition workflow"""
    self.pending_actions[self.whc.progress_messenger0.recipient_phone] = {
        'type': 'compose_email',
        'step': 'subject',
        'draft': {'attachments': []}
    }
    return {
        'type': 'quick_reply',
        'text': "📝 Let's compose an email\n\nSubject:",
        'options': {'cancel': '❌ Cancel Composition'}
    }

start_event_create(message) async

Initiate event creation workflow

Source code in toolboxv2/mods/WhatsAppTb/client.py

async def start_event_create(self, message):
    """Initiate event creation workflow"""
    self.pending_actions[self.whc.progress_messenger0.recipient_phone] = {
        'type': 'create_event',
        'step': 'title',
        'event_data': {}
    }
    return {
        'type': 'quick_reply',
        'text': "Let's create an event! What's the title?",
        'options': {'cancel': '❌ Cancel'}
    }

stop_agent(*b) async

Stop the currently running agent

Source code in toolboxv2/mods/WhatsAppTb/client.py

async def stop_agent(self, *b):
    """Stop the currently running agent"""
    if self.agent:
        self.agent.stop()
        return True
    return False

system_task(message) async

Initiate email search workflow

Source code in toolboxv2/mods/WhatsAppTb/client.py

async def system_task(self, message):
    """Initiate email search workflow"""
    self.pending_actions[self.whc.progress_messenger0.recipient_phone] = {
        'type': 'system',
        'step': 'await_query'
    }
    return {
        'type': 'quick_reply',
        'text': "Now prompt the 🧠ISAA-System 📝",
        'options': {'cancel': '❌ Cancel Search'}
    }

server

AppManager

Source code in toolboxv2/mods/WhatsAppTb/server.py

class AppManager(metaclass=Singleton):
    pepper = "pepper0"

    def __init__(self, start_port: int = 8000, port_range: int = 10, em=None):
        self.instances: dict[str, dict] = {}
        self.start_port = start_port
        self.port_range = port_range
        self.threads: dict[str, Thread] = {}
        self.stop_events: dict[str, Event] = {}
        self.message_queue: asyncio.Queue = asyncio.Queue()
        self.last_messages: dict[str, datetime] = {}
        self.keys: dict[str, str] = {}
        self.forwarders: dict[str, dict] = {}
        self.runner = lambda :None

        if em is None:
            from toolboxv2 import get_app
            em = get_app().get_mod("EventManager")
        from toolboxv2.mods import EventManager
        self.event_manager: EventManager = em.get_manager()

        # Set up signal handlers for graceful shutdown
        try:
            if threading.current_thread() is threading.main_thread():
                signal.signal(signal.SIGINT, self.signal_handler)
                signal.signal(signal.SIGTERM, self.signal_handler)
        except Exception:
            pass

    def offline(self, instance_id):

        def mark_as_offline():
            self.forwarders[instance_id]['send'] = None
            return 'done'

        return mark_as_offline

    def online(self, instance_id):

        def mark_as_online():
            return self.instances[instance_id]['app']

        def set_callbacks(callback, e_callback=None):
            if callback is not None:
                self.forwarders[instance_id]['send'] = callback
            if e_callback is not None:
                self.forwarders[instance_id]['sende'] = e_callback

        return mark_as_online(), set_callbacks

    def get_next_available_port(self) -> int:
        """Find the next available port in the range."""
        used_ports = {instance['port'] for instance in self.instances.values()}
        for port in range(self.start_port, self.start_port + self.port_range):
            if port not in used_ports:
                return port
        raise RuntimeError("No available ports in range")

    def add_instance(self, instance_id: str, **kwargs):
        """
        Add a new app instance to the manager with automatic port assignment.
        """
        if instance_id in self.instances:
            raise ValueError(f"Instance {instance_id} already exists")

        port = self.get_next_available_port()
        app_instance = WhatsApp(**kwargs)

        self.instances[instance_id] = {
            'app': app_instance,
            'port': port,
            'kwargs': kwargs,
            'phone_number_id': kwargs.get("phone_number_id", {}),
            'retry_count': 0,
            'max_retries': 3,
            'retry_delay': 5
        }
        self.keys[instance_id] = Code.one_way_hash(kwargs.get("phone_number_id", {}).get("key"), "WhatsappAppManager",
                                                   self.pepper)
        self.forwarders[instance_id] = {}

        # Set up message handlers
        @app_instance.on_message
        async def message_handler(message):
            await self.on_message(instance_id, message)

        @app_instance.on_event
        async def event_handler(event):
            await self.on_event(instance_id, event)

        @app_instance.on_verification
        async def verification_handler(verification):
            await self.on_verification(instance_id, verification)

        # Create stop event for this instance Error parsing message1:
        self.stop_events[instance_id] = Event()

    def run_instance(self, instance_id: str):
        """Run a single instance in a separate thread with error handling and automatic restart."""
        instance_data = self.instances[instance_id]
        stop_event = self.stop_events[instance_id]

        while not stop_event.is_set():
            try:
                logger.info(f"Starting instance {instance_id} on port {instance_data['port']}")
                instance_data['app'].run(host='0.0.0.0', port=instance_data['port'])

            except Exception as e:
                logger.error(f"Error in instance {instance_id}: {str(e)}")
                instance_data['retry_count'] += 1

                if instance_data['retry_count'] > instance_data['max_retries']:
                    logger.error(f"Max retries exceeded for instance {instance_id}")
                    break

                logger.info(f"Restarting instance {instance_id} in {instance_data['retry_delay']} seconds...")
                time.sleep(instance_data['retry_delay'])

                # Recreate the instance
                instance_data['app'] = WhatsApp(**instance_data['kwargs'])
                continue

    async def on_message(self, instance_id: str, message: Message):
        """Handle and forward incoming messages."""
        logger.info(f"Message from instance {instance_id}: {message}")
        if instance_id in self.forwarders and 'send' in self.forwarders[instance_id]:
            await self.forwarders[instance_id]['send'](message)

    async def on_event(self, instance_id: str, event):
        """Handle events."""
        logger.info(f"Event from instance {instance_id}: {event}")
        if instance_id in self.forwarders and 'sende' in self.forwarders[instance_id] and self.forwarders[instance_id]['sende'] is not None:
            self.forwarders[instance_id]['sende'](event)

    async def on_verification(self, instance_id: str, verification):
        """Handle verification events."""
        logger.info(f"Verification from instance {instance_id}: {verification}")

    def run_all_instances(self):
        """Start all instances in separate daemon threads."""
        # Start message forwarder

        # Start all instances
        for instance_id in self.instances:
            thread = Thread(
                target=self.run_instance,
                args=(instance_id,),
                daemon=True,
                name=f"WhatsApp-{instance_id}"
            )
            self.threads[instance_id] = thread
            thread.start()

    def signal_handler(self, signum, frame):
        """Handle shutdown signals gracefully."""
        logger.info("Shutdown signal received, stopping all instances...")
        self.stop_all_instances()
        sys.exit(0)

    def stop_all_instances(self):
        """Stop all running instances gracefully."""
        for instance_id in self.stop_events:
            self.stop_events[instance_id].set()

        for thread in self.threads.values():
            thread.join(timeout=5)

    def create_manager_ui(self, start_assistant):
        """Enhanced WhatsApp Manager UI with instance configuration controls"""
        self.runner = start_assistant
        def ui_manager():
            # Track instance states and messages
            original_on_message = self.on_message

            async def enhanced_on_message(instance_id: str, message):
                self.last_messages[instance_id] = datetime.now()
                await original_on_message(instance_id, message)

            self.on_message = enhanced_on_message

            def create_instance_card(instance_id: str):
                """Interactive instance control card"""
                config = self.instances[instance_id]
                with ui.card().classes('w-full p-4 mb-4 bg-gray-50 dark:bg-gray-800').style("background-color: var(--background-color) !important"):
                    # Header Section
                    with ui.row().classes('w-full justify-between items-center'):
                        ui.label(f'📱 {instance_id}').classes('text-xl font-bold')

                        # Status Indicator
                        ui.label().bind_text_from(
                            self.threads, instance_id,
                            lambda x: 'Running' if x and x.is_alive() else 'Stopped'
                        )

                    # Configuration Display
                    with ui.grid(columns=2).classes('w-full mt-4 gap-2'):

                        ui.label('port:').classes('font-bold')
                        ui.label(config['port'])

                        ui.label('Last Activity:').classes('font-bold')
                        ui.label().bind_text_from(
                            self.last_messages, instance_id,
                            lambda x: x.strftime("%Y-%m-%d %H:%M:%S") if x else 'Never'
                        )

                    # Action Controls
                    with ui.row().classes('w-full mt-4 gap-2'):
                        with ui.button(icon='settings', on_click=lambda: edit_dialog.open()).props('flat'):
                            ui.tooltip('Configure')

                        with ui.button(icon='refresh', color='orange',
                                       on_click=lambda: self.restart_instance(instance_id)):
                            ui.tooltip('Restart')

                        with ui.button(icon='stop', color='red',
                                       on_click=lambda: self.stop_instance(instance_id)):
                            ui.tooltip('Stop')

                    # Edit Configuration Dialog
                    with ui.dialog() as edit_dialog, ui.card().classes('p-4 gap-4'):
                        new_key = ui.input('API Key', value=config['phone_number_id'].get('key', ''))
                        new_number = ui.input('Phone Number', value=config['phone_number_id'].get('number', ''))

                        with ui.row().classes('w-full justify-end'):
                            ui.button('Cancel', on_click=edit_dialog.close)
                            ui.button('Save', color='primary', on_click=lambda: (
                                self.update_instance_config(
                                    instance_id,
                                    new_key.value,
                                    new_number.value
                                ),
                                edit_dialog.close()
                            ))

            # Main UI Layout
            with ui.column().classes('w-full max-w-4xl mx-auto p-4'):
                ui.label('WhatsApp Instance Manager').classes('text-2xl font-bold mb-6')

                # Add Instance Section
                with ui.expansion('➕ Add New Instance', icon='add').classes('w-full'):
                    with ui.card().classes('w-full p-4 mt-2'):
                        instance_id = ui.input('Instance ID').classes('w-full')
                        token = ui.input('API Token').classes('w-full')
                        phone_key = ui.input('Phone Number Key').classes('w-full')
                        phone_number = ui.input('Phone Number').classes('w-full')

                        with ui.row().classes('w-full justify-end gap-2'):
                            ui.button('Clear', on_click=lambda: (
                                instance_id.set_value(''),
                                token.set_value(''),
                                phone_key.set_value(''),
                                phone_number.set_value('')
                            ))
                            ui.button('Create', color='positive', on_click=lambda: (
                                self.add_update_instance(
                                    instance_id.value,
                                    token.value,
                                    phone_key.value,
                                    phone_number.value
                                ),
                                instances_container.refresh()
                            ))

                # Instances Display
                instances_container = ui.column().classes('w-full')
                with instances_container:
                    for instance_id in self.instances:
                        create_instance_card(instance_id)

        return ui_manager

    # Add to manager class
    def add_update_instance(self, instance_id, token, phone_key, phone_number):
        """Add or update instance configuration"""
        if instance_id in self.instances:
            self.stop_instance(instance_id)
            del self.instances[instance_id]

        self.add_instance(
            instance_id,
            token=token,
            phone_number_id={
                'key': phone_key,
                'number': phone_number
            },
            verify_token=os.getenv("WHATSAPP_VERIFY_TOKEN")
        )
        self.start_instance(instance_id)

    def update_instance_config(self, instance_id, new_key, new_number):
        """Update existing instance configuration"""
        if instance_id in self.instances:
            self.instances[instance_id]['phone_number_id'] = {
                'key': new_key,
                'number': new_number
            }
            self.restart_instance(instance_id)

    def restart_instance(self, instance_id):
        """Safe restart of instance"""
        self.stop_instance(instance_id)
        self.start_instance(instance_id)

    def stop_instance(self, instance_id):
        """Graceful stop of instance"""
        if instance_id in self.threads:
            self.stop_events[instance_id].set()
            self.threads[instance_id].join(timeout=5)
            del self.threads[instance_id]

    def start_instance(self, instance_id):
        """Start instance thread"""
        print("Starting Istance")

        self.stop_events[instance_id] = threading.Event()
        self.threads[instance_id] = threading.Thread(
            target=self.run_instance,
            args=(instance_id,),
            daemon=True
        )
        self.threads[instance_id].start()
        print("Running starter", self.runner())

add_instance(instance_id, **kwargs)

Add a new app instance to the manager with automatic port assignment.

Source code in toolboxv2/mods/WhatsAppTb/server.py

def add_instance(self, instance_id: str, **kwargs):
    """
    Add a new app instance to the manager with automatic port assignment.
    """
    if instance_id in self.instances:
        raise ValueError(f"Instance {instance_id} already exists")

    port = self.get_next_available_port()
    app_instance = WhatsApp(**kwargs)

    self.instances[instance_id] = {
        'app': app_instance,
        'port': port,
        'kwargs': kwargs,
        'phone_number_id': kwargs.get("phone_number_id", {}),
        'retry_count': 0,
        'max_retries': 3,
        'retry_delay': 5
    }
    self.keys[instance_id] = Code.one_way_hash(kwargs.get("phone_number_id", {}).get("key"), "WhatsappAppManager",
                                               self.pepper)
    self.forwarders[instance_id] = {}

    # Set up message handlers
    @app_instance.on_message
    async def message_handler(message):
        await self.on_message(instance_id, message)

    @app_instance.on_event
    async def event_handler(event):
        await self.on_event(instance_id, event)

    @app_instance.on_verification
    async def verification_handler(verification):
        await self.on_verification(instance_id, verification)

    # Create stop event for this instance Error parsing message1:
    self.stop_events[instance_id] = Event()

add_update_instance(instance_id, token, phone_key, phone_number)

Add or update instance configuration

Source code in toolboxv2/mods/WhatsAppTb/server.py

def add_update_instance(self, instance_id, token, phone_key, phone_number):
    """Add or update instance configuration"""
    if instance_id in self.instances:
        self.stop_instance(instance_id)
        del self.instances[instance_id]

    self.add_instance(
        instance_id,
        token=token,
        phone_number_id={
            'key': phone_key,
            'number': phone_number
        },
        verify_token=os.getenv("WHATSAPP_VERIFY_TOKEN")
    )
    self.start_instance(instance_id)

create_manager_ui(start_assistant)

Enhanced WhatsApp Manager UI with instance configuration controls

Source code in toolboxv2/mods/WhatsAppTb/server.py

def create_manager_ui(self, start_assistant):
    """Enhanced WhatsApp Manager UI with instance configuration controls"""
    self.runner = start_assistant
    def ui_manager():
        # Track instance states and messages
        original_on_message = self.on_message

        async def enhanced_on_message(instance_id: str, message):
            self.last_messages[instance_id] = datetime.now()
            await original_on_message(instance_id, message)

        self.on_message = enhanced_on_message

        def create_instance_card(instance_id: str):
            """Interactive instance control card"""
            config = self.instances[instance_id]
            with ui.card().classes('w-full p-4 mb-4 bg-gray-50 dark:bg-gray-800').style("background-color: var(--background-color) !important"):
                # Header Section
                with ui.row().classes('w-full justify-between items-center'):
                    ui.label(f'📱 {instance_id}').classes('text-xl font-bold')

                    # Status Indicator
                    ui.label().bind_text_from(
                        self.threads, instance_id,
                        lambda x: 'Running' if x and x.is_alive() else 'Stopped'
                    )

                # Configuration Display
                with ui.grid(columns=2).classes('w-full mt-4 gap-2'):

                    ui.label('port:').classes('font-bold')
                    ui.label(config['port'])

                    ui.label('Last Activity:').classes('font-bold')
                    ui.label().bind_text_from(
                        self.last_messages, instance_id,
                        lambda x: x.strftime("%Y-%m-%d %H:%M:%S") if x else 'Never'
                    )

                # Action Controls
                with ui.row().classes('w-full mt-4 gap-2'):
                    with ui.button(icon='settings', on_click=lambda: edit_dialog.open()).props('flat'):
                        ui.tooltip('Configure')

                    with ui.button(icon='refresh', color='orange',
                                   on_click=lambda: self.restart_instance(instance_id)):
                        ui.tooltip('Restart')

                    with ui.button(icon='stop', color='red',
                                   on_click=lambda: self.stop_instance(instance_id)):
                        ui.tooltip('Stop')

                # Edit Configuration Dialog
                with ui.dialog() as edit_dialog, ui.card().classes('p-4 gap-4'):
                    new_key = ui.input('API Key', value=config['phone_number_id'].get('key', ''))
                    new_number = ui.input('Phone Number', value=config['phone_number_id'].get('number', ''))

                    with ui.row().classes('w-full justify-end'):
                        ui.button('Cancel', on_click=edit_dialog.close)
                        ui.button('Save', color='primary', on_click=lambda: (
                            self.update_instance_config(
                                instance_id,
                                new_key.value,
                                new_number.value
                            ),
                            edit_dialog.close()
                        ))

        # Main UI Layout
        with ui.column().classes('w-full max-w-4xl mx-auto p-4'):
            ui.label('WhatsApp Instance Manager').classes('text-2xl font-bold mb-6')

            # Add Instance Section
            with ui.expansion('➕ Add New Instance', icon='add').classes('w-full'):
                with ui.card().classes('w-full p-4 mt-2'):
                    instance_id = ui.input('Instance ID').classes('w-full')
                    token = ui.input('API Token').classes('w-full')
                    phone_key = ui.input('Phone Number Key').classes('w-full')
                    phone_number = ui.input('Phone Number').classes('w-full')

                    with ui.row().classes('w-full justify-end gap-2'):
                        ui.button('Clear', on_click=lambda: (
                            instance_id.set_value(''),
                            token.set_value(''),
                            phone_key.set_value(''),
                            phone_number.set_value('')
                        ))
                        ui.button('Create', color='positive', on_click=lambda: (
                            self.add_update_instance(
                                instance_id.value,
                                token.value,
                                phone_key.value,
                                phone_number.value
                            ),
                            instances_container.refresh()
                        ))

            # Instances Display
            instances_container = ui.column().classes('w-full')
            with instances_container:
                for instance_id in self.instances:
                    create_instance_card(instance_id)

    return ui_manager

get_next_available_port()

Find the next available port in the range.

Source code in toolboxv2/mods/WhatsAppTb/server.py

def get_next_available_port(self) -> int:
    """Find the next available port in the range."""
    used_ports = {instance['port'] for instance in self.instances.values()}
    for port in range(self.start_port, self.start_port + self.port_range):
        if port not in used_ports:
            return port
    raise RuntimeError("No available ports in range")

on_event(instance_id, event) async

Handle events.

Source code in toolboxv2/mods/WhatsAppTb/server.py

async def on_event(self, instance_id: str, event):
    """Handle events."""
    logger.info(f"Event from instance {instance_id}: {event}")
    if instance_id in self.forwarders and 'sende' in self.forwarders[instance_id] and self.forwarders[instance_id]['sende'] is not None:
        self.forwarders[instance_id]['sende'](event)

on_message(instance_id, message) async

Handle and forward incoming messages.

Source code in toolboxv2/mods/WhatsAppTb/server.py

async def on_message(self, instance_id: str, message: Message):
    """Handle and forward incoming messages."""
    logger.info(f"Message from instance {instance_id}: {message}")
    if instance_id in self.forwarders and 'send' in self.forwarders[instance_id]:
        await self.forwarders[instance_id]['send'](message)

on_verification(instance_id, verification) async

Handle verification events.

Source code in toolboxv2/mods/WhatsAppTb/server.py

async def on_verification(self, instance_id: str, verification):
    """Handle verification events."""
    logger.info(f"Verification from instance {instance_id}: {verification}")

restart_instance(instance_id)

Safe restart of instance

Source code in toolboxv2/mods/WhatsAppTb/server.py

def restart_instance(self, instance_id):
    """Safe restart of instance"""
    self.stop_instance(instance_id)
    self.start_instance(instance_id)

run_all_instances()

Start all instances in separate daemon threads.

Source code in toolboxv2/mods/WhatsAppTb/server.py

def run_all_instances(self):
    """Start all instances in separate daemon threads."""
    # Start message forwarder

    # Start all instances
    for instance_id in self.instances:
        thread = Thread(
            target=self.run_instance,
            args=(instance_id,),
            daemon=True,
            name=f"WhatsApp-{instance_id}"
        )
        self.threads[instance_id] = thread
        thread.start()

run_instance(instance_id)

Run a single instance in a separate thread with error handling and automatic restart.

Source code in toolboxv2/mods/WhatsAppTb/server.py

def run_instance(self, instance_id: str):
    """Run a single instance in a separate thread with error handling and automatic restart."""
    instance_data = self.instances[instance_id]
    stop_event = self.stop_events[instance_id]

    while not stop_event.is_set():
        try:
            logger.info(f"Starting instance {instance_id} on port {instance_data['port']}")
            instance_data['app'].run(host='0.0.0.0', port=instance_data['port'])

        except Exception as e:
            logger.error(f"Error in instance {instance_id}: {str(e)}")
            instance_data['retry_count'] += 1

            if instance_data['retry_count'] > instance_data['max_retries']:
                logger.error(f"Max retries exceeded for instance {instance_id}")
                break

            logger.info(f"Restarting instance {instance_id} in {instance_data['retry_delay']} seconds...")
            time.sleep(instance_data['retry_delay'])

            # Recreate the instance
            instance_data['app'] = WhatsApp(**instance_data['kwargs'])
            continue

signal_handler(signum, frame)

Handle shutdown signals gracefully.

Source code in toolboxv2/mods/WhatsAppTb/server.py

def signal_handler(self, signum, frame):
    """Handle shutdown signals gracefully."""
    logger.info("Shutdown signal received, stopping all instances...")
    self.stop_all_instances()
    sys.exit(0)

start_instance(instance_id)

Start instance thread

Source code in toolboxv2/mods/WhatsAppTb/server.py

def start_instance(self, instance_id):
    """Start instance thread"""
    print("Starting Istance")

    self.stop_events[instance_id] = threading.Event()
    self.threads[instance_id] = threading.Thread(
        target=self.run_instance,
        args=(instance_id,),
        daemon=True
    )
    self.threads[instance_id].start()
    print("Running starter", self.runner())

stop_all_instances()

Stop all running instances gracefully.

Source code in toolboxv2/mods/WhatsAppTb/server.py

def stop_all_instances(self):
    """Stop all running instances gracefully."""
    for instance_id in self.stop_events:
        self.stop_events[instance_id].set()

    for thread in self.threads.values():
        thread.join(timeout=5)

stop_instance(instance_id)

Graceful stop of instance

Source code in toolboxv2/mods/WhatsAppTb/server.py

def stop_instance(self, instance_id):
    """Graceful stop of instance"""
    if instance_id in self.threads:
        self.stop_events[instance_id].set()
        self.threads[instance_id].join(timeout=5)
        del self.threads[instance_id]

update_instance_config(instance_id, new_key, new_number)

Update existing instance configuration

Source code in toolboxv2/mods/WhatsAppTb/server.py

def update_instance_config(self, instance_id, new_key, new_number):
    """Update existing instance configuration"""
    if instance_id in self.instances:
        self.instances[instance_id]['phone_number_id'] = {
            'key': new_key,
            'number': new_number
        }
        self.restart_instance(instance_id)

utils

ProgressMessenger

Source code in toolboxv2/mods/WhatsAppTb/utils.py

class ProgressMessenger:
    def __init__(self, messenger, recipient_phone: str, max_steps: int = 5, emoji_set: list[str] = None, content=None):
        self.messenger = messenger
        self.recipient_phone = recipient_phone
        self.max_steps = max_steps
        self.emoji_set = emoji_set or ["⬜", "⬛", "🟩", "🟨", "🟦"]
        self.message_id = None
        self.content = content

    def send_initial_message(self, mode: str = "progress"):
        """
        Sends the initial message. Modes can be 'progress' or 'loading'.
        """
        if mode == "progress":
            emoji_legend = "\n".join(
                f"{emoji} - Step {i + 1}" for i, emoji in enumerate(self.emoji_set)
            )
            content = (
                "Progress is being updated in real-time!\n\n"
                "Legend:\n"
                f"{emoji_legend}\n\n"
                "Stay tuned for updates!"
            )
        elif mode == "loading":
            content = (
                "Loading in progress! 🌀\n"
                "The indicator will loop until work is done."
            )
        else:
            raise ValueError("Invalid mode. Use 'progress' or 'loading'.")

        if self.content is not None:
            content += '\n'+self.content
        message = self.messenger.create_message(content=content, to=self.recipient_phone)
        response = message.send(sender=0)
        self.message_id = response.get("messages", [{}])[0].get("id")
        logging.info(f"Initial message sent: {content}")
        return self.message_id

    def update_progress(self, step_flag: threading.Event):
        """
        Updates the reaction on the message to represent progress.
        """
        if not self.message_id:
            raise ValueError("Message ID not found. Ensure the initial message is sent first.")
        message = self.messenger.create_message(id=self.message_id, to=self.recipient_phone)
        for step in range(self.max_steps):
            emoji = self.emoji_set[step % len(self.emoji_set)]
            message.react(emoji)
            logging.info(f"Progress updated: Step {step + 1}/{self.max_steps} with emoji {emoji}")
            while not step_flag.is_set():
                time.sleep(0.5)
            step_flag.clear()
        # Final acknowledgment
        message.react("👍")
        logging.info("Progress completed with final acknowledgment.")

    def update_loading(self, stop_flag: threading.Event):
        """
        Continuously updates the reaction to represent a looping 'loading' indicator.
        """
        if not self.message_id:
            raise ValueError("Message ID not found. Ensure the initial message is sent first.")
        message = self.messenger.create_message(id=self.message_id, to=self.recipient_phone)
        step = 0
        while not stop_flag.is_set():
            emoji = self.emoji_set[step % len(self.emoji_set)]
            message.react(emoji)
            logging.info(f"Loading update: {emoji}")
            time.sleep(1)  # Faster updates for loading
            step += 1
        # Final acknowledgment
        message.react("✅")
        logging.info("Loading completed with final acknowledgment.")
        message.reply("✅Done✅")

    def start_progress_in_background(self, step_flag):
        """
        Starts the progress update in a separate thread.
        """
        threading.Thread(target=self.update_progress, args=(step_flag, ), daemon=True).start()

    def start_loading_in_background(self, stop_flag: threading.Event):
        """
        Starts the loading update in a separate thread.
        """
        threading.Thread(target=self.update_loading, args=(stop_flag,), daemon=True).start()

send_initial_message(mode='progress')

Sends the initial message. Modes can be 'progress' or 'loading'.

Source code in toolboxv2/mods/WhatsAppTb/utils.py

def send_initial_message(self, mode: str = "progress"):
    """
    Sends the initial message. Modes can be 'progress' or 'loading'.
    """
    if mode == "progress":
        emoji_legend = "\n".join(
            f"{emoji} - Step {i + 1}" for i, emoji in enumerate(self.emoji_set)
        )
        content = (
            "Progress is being updated in real-time!\n\n"
            "Legend:\n"
            f"{emoji_legend}\n\n"
            "Stay tuned for updates!"
        )
    elif mode == "loading":
        content = (
            "Loading in progress! 🌀\n"
            "The indicator will loop until work is done."
        )
    else:
        raise ValueError("Invalid mode. Use 'progress' or 'loading'.")

    if self.content is not None:
        content += '\n'+self.content
    message = self.messenger.create_message(content=content, to=self.recipient_phone)
    response = message.send(sender=0)
    self.message_id = response.get("messages", [{}])[0].get("id")
    logging.info(f"Initial message sent: {content}")
    return self.message_id

start_loading_in_background(stop_flag)

Starts the loading update in a separate thread.

Source code in toolboxv2/mods/WhatsAppTb/utils.py

def start_loading_in_background(self, stop_flag: threading.Event):
    """
    Starts the loading update in a separate thread.
    """
    threading.Thread(target=self.update_loading, args=(stop_flag,), daemon=True).start()

start_progress_in_background(step_flag)

Starts the progress update in a separate thread.

Source code in toolboxv2/mods/WhatsAppTb/utils.py

def start_progress_in_background(self, step_flag):
    """
    Starts the progress update in a separate thread.
    """
    threading.Thread(target=self.update_progress, args=(step_flag, ), daemon=True).start()

update_loading(stop_flag)

Continuously updates the reaction to represent a looping 'loading' indicator.

Source code in toolboxv2/mods/WhatsAppTb/utils.py

def update_loading(self, stop_flag: threading.Event):
    """
    Continuously updates the reaction to represent a looping 'loading' indicator.
    """
    if not self.message_id:
        raise ValueError("Message ID not found. Ensure the initial message is sent first.")
    message = self.messenger.create_message(id=self.message_id, to=self.recipient_phone)
    step = 0
    while not stop_flag.is_set():
        emoji = self.emoji_set[step % len(self.emoji_set)]
        message.react(emoji)
        logging.info(f"Loading update: {emoji}")
        time.sleep(1)  # Faster updates for loading
        step += 1
    # Final acknowledgment
    message.react("✅")
    logging.info("Loading completed with final acknowledgment.")
    message.reply("✅Done✅")

update_progress(step_flag)

Updates the reaction on the message to represent progress.

Source code in toolboxv2/mods/WhatsAppTb/utils.py

def update_progress(self, step_flag: threading.Event):
    """
    Updates the reaction on the message to represent progress.
    """
    if not self.message_id:
        raise ValueError("Message ID not found. Ensure the initial message is sent first.")
    message = self.messenger.create_message(id=self.message_id, to=self.recipient_phone)
    for step in range(self.max_steps):
        emoji = self.emoji_set[step % len(self.emoji_set)]
        message.react(emoji)
        logging.info(f"Progress updated: Step {step + 1}/{self.max_steps} with emoji {emoji}")
        while not step_flag.is_set():
            time.sleep(0.5)
        step_flag.clear()
    # Final acknowledgment
    message.react("👍")
    logging.info("Progress completed with final acknowledgment.")

cli_functions

replace_bracketed_content(text, replacements, inlist=False)

Ersetzt Inhalte in eckigen Klammern mit entsprechenden Werten aus einem Wörterbuch.

:param text: Der zu verarbeitende Text als String. :param replacements: Ein Wörterbuch mit Schlüssel-Wert-Paaren für die Ersetzung. :return: Den modifizierten Text.

Source code in toolboxv2/mods/cli_functions.py

def replace_bracketed_content(text, replacements, inlist=False):
    """
    Ersetzt Inhalte in eckigen Klammern mit entsprechenden Werten aus einem Wörterbuch.

    :param text: Der zu verarbeitende Text als String.
    :param replacements: Ein Wörterbuch mit Schlüssel-Wert-Paaren für die Ersetzung.
    :return: Den modifizierten Text.
    """
    # Finde alle Vorkommen von Texten in eckigen Klammern
    matches = re.findall(r'\[([^\]]+)\]', text)

    # Ersetze jeden gefundenen Text durch den entsprechenden Wert aus dem Wörterbuch
    as_list = text.split(' ')
    i = 0
    for key in matches:
        if key in replacements:
            if not inlist:
                text = text.replace(f'[{key}]', str(replacements[key]))
            else:
                as_list[i] = replacements[key]
        i += 1
    if not inlist:
        return text
    return as_list

helper

create_invitation(app, username)

Creates a one-time invitation code for a user to link a new device.

Source code in toolboxv2/mods/helper.py

@export(mod_name=Name, name="create-invitation", test=False)
def create_invitation(app: App, username: str):
    """Creates a one-time invitation code for a user to link a new device."""
    print(f"Creating invitation for user '{username}'...")
    app.load_mod("CloudM")
    result = app.run_any(TBEF.CLOUDM_AUTHMANAGER.GET_INVITATION,
                         get_results=True,
                         username=username)

    if result.is_ok():
        print(f"✅ Invitation code for '{username}': {result.get()}")
    else:
        print("❌ Error creating invitation:")
        result.print()
    return result

create_user(app, username, email)

Creates a new user with a generated key pair.

Source code in toolboxv2/mods/helper.py

@export(mod_name=Name, name="create-user", test=False)
def create_user(app: App, username: str, email: str):
    """Creates a new user with a generated key pair."""
    print(f"Creating user '{username}' with email '{email}'...")
    app.load_mod("CloudM")
    # Generate an invitation on the fly
    invitation_res = app.run_any(TBEF.CLOUDM_AUTHMANAGER.GET_INVITATION,
                                 get_results=True,
                                 username=username)
    if invitation_res.is_error():
        print("❌ Error creating invitation:")
        invitation_res.print()
        return invitation_res

    result = app.run_any(TBEF.CLOUDM_AUTHMANAGER.CRATE_LOCAL_ACCOUNT,
                         get_results=True,
                         username=username,
                         email=email,
                         invitation=invitation_res.get(),
                         create=True)

    if result.is_ok():
        print(f"✅ User '{username}' created successfully.")
    else:
        print("❌ Error creating user:")
        result.print()
    return result

delete_user_cli(app, username)

Deletes a user and all their associated data.

Source code in toolboxv2/mods/helper.py

@export(mod_name=Name, name="delete-user", test=False)
def delete_user_cli(app: App, username: str):
    """Deletes a user and all their associated data."""
    print(f"Attempting to delete user '{username}'...")
    app.load_mod("CloudM")
    result = app.run_any(TBEF.CLOUDM_AUTHMANAGER.DELETE_USER, get_results=True, username=username)

    if result.is_ok():
        print(f"✅ User '{username}' has been deleted.")
    else:
        print(f"❌ Error deleting user: {result.info.get('help_text')}")
    return result

init_system(app) async

Initializes the ToolBoxV2 system by creating the first administrative user. This is an interactive command.

Source code in toolboxv2/mods/helper.py

@export(mod_name=Name, name="init_system", test=False)
async def init_system(app: App):
    """
    Initializes the ToolBoxV2 system by creating the first administrative user.
    This is an interactive command.
    """
    print("--- ToolBoxV2 System Initialization ---")
    print("This will guide you through creating the first administrator account.")
    print("This account will have the highest permission level.\n")

    try:
        username = input("Enter the administrator's username: ").strip()
        if not username:
            print("Username cannot be empty.")
            return Result.default_user_error("Username cannot be empty.")

        email = input(f"Enter the email for '{username}': ").strip()
        if not email: # A simple check, can be improved with regex
            print("Email cannot be empty.")
            return Result.default_user_error("Email cannot be empty.")

        print(f"\nCreating user '{username}' with email '{email}'...")
        # Call the internal function to create the account
        # The 'create=True' flag likely handles the initial key generation
        result = await app.a_run_any(TBEF.CLOUDM.REGISTER_INITIAL_LOOT_USER,
                                     user_name=username,
                                     email=email,
                                     get_results=True)

        if result.is_ok():
            print("\n✅ Administrator account created successfully!")
            print("   A new cryptographic key pair has been generated for this user.")
            print("   Authentication is handled automatically using these keys.")
            print("   You can now use other CLI commands or log into the web UI.")
            return Result.ok("System initialized successfully.")
        else:
            print("\n❌ Error creating administrator account:")
            result.print()
            return result

    except (KeyboardInterrupt, EOFError):
        print("\n\nInitialization cancelled by user.")
        return Result.default_user_error("Initialization cancelled.")
    except Exception as e:
        print(f"\nAn unexpected error occurred: {e}")
        return Result.default_internal_error(f"An unexpected error occurred: {e}")

list_users_cli(app)

Lists all registered users.

Source code in toolboxv2/mods/helper.py

@export(mod_name=Name, name="list-users", test=False)
def list_users_cli(app: App):
    """Lists all registered users."""
    print("Fetching user list...")
    app.load_mod("CloudM")
    result = app.run_any(TBEF.CLOUDM_AUTHMANAGER.LIST_USERS, get_results=True)

    if result.is_ok():
        users = result.get()
        if not users:
            print("No users found.")
            return result

        print("--- Registered Users ---")
        # Simple table formatting
        print(f"{'Username':<25} {'Email':<30} {'Level'}")
        print("------------------------")
        for user in users:
            print(f"{user['username']:<25} {user['email']:<30} {user['level']}")
        print("------------------------")
    else:
        print("❌ Error listing users:")
        result.print()

    return result

send_magic_link(app, username)

Sends a magic login link to the user's registered email address.

Source code in toolboxv2/mods/helper.py

@export(mod_name=Name, name="send-magic-link", test=False)
def send_magic_link(app: App, username: str):
    """Sends a magic login link to the user's registered email address."""
    print(f"Sending magic link to user '{username}'...")
    app.load_mod("CloudM")
    result = app.run_any(TBEF.CLOUDM_AUTHMANAGER.GET_MAGIC_LINK_EMAIL,
                         get_results=True,
                         username=username)

    if result.is_ok():
        print(f"✅ Magic link sent successfully to the email address associated with '{username}'.")
    else:
        print("❌ Error sending magic link:")
        result.print()
    return result

isaa

CodingAgent

live

AsyncCodeDetector

Bases: NodeVisitor

Detect async code and top-level await

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

class AsyncCodeDetector(ast.NodeVisitor):
    """Detect async code and top-level await"""
    def __init__(self):
        self.has_async = False
        self.has_top_level_await = False
        self.await_nodes = []

    def visit_AsyncFunctionDef(self, node):
        self.has_async = True
        self.generic_visit(node)

    def visit_Await(self, node):
        self.has_async = True
        # Track all await nodes
        self.await_nodes.append(node)
        # Check if this await is at top level
        parent = node
        while hasattr(parent, 'parent'):
            parent = parent.parent
            if isinstance(parent, ast.AsyncFunctionDef | ast.FunctionDef):
                break
        else:
            self.has_top_level_await = True
        self.generic_visit(node)

CargoRustInterface

Usage :

Create interface

cargo_interface = CargoRustInterface()

Set up new project

await cargo_interface.setup_project("hello_rust")

Add a dependency

await cargo_interface.add_dependency("serde", "1.0")

Write and run some code

code = """ fn main() { println!("Hello, Rust!"); } """ result = await cargo_interface.run_code(code)

Modify code

new_function = """ fn main() { println!("Modified Hello, Rust!"); } """ await cargo_interface.modify_code(new_function, "main()")

Build and test

await cargo_interface.build() await cargo_interface.test()

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

class CargoRustInterface:
    '''Usage :
# Create interface
cargo_interface = CargoRustInterface()

# Set up new project
await cargo_interface.setup_project("hello_rust")

# Add a dependency
await cargo_interface.add_dependency("serde", "1.0")

# Write and run some code
code = """
fn main() {
    println!("Hello, Rust!");
}
"""
result = await cargo_interface.run_code(code)

# Modify code
new_function = """
fn main() {
    println!("Modified Hello, Rust!");
}
"""
await cargo_interface.modify_code(new_function, "main()")

# Build and test
await cargo_interface.build()
await cargo_interface.test()

    '''
    def __init__(self, session_dir=None, auto_remove=True):
        """Initialize the Rust/Cargo interface"""
        self.auto_remove = auto_remove
        self._session_dir = session_dir or Path.home() / '.cargo_sessions'
        self._session_dir.mkdir(exist_ok=True)
        self.vfs = VirtualFileSystem(self._session_dir / 'virtual_fs')
        self.output_history = {}
        self._execution_count = 0
        self.current_project = None

    def reset(self):
        """Reset the interface state"""
        if self.auto_remove and self.current_project:
            shutil.rmtree(self.current_project, ignore_errors=True)
        self.output_history.clear()
        self._execution_count = 0
        self.current_project = None

    async def setup_project(self, name: str) -> str:
        """Set up a new Cargo project"""
        try:
            project_path = self.vfs.base_dir / name
            if project_path.exists():
                shutil.rmtree(project_path)

            result = subprocess.run(
                ['cargo', 'new', str(project_path)],
                capture_output=True,
                text=True, check=True
            )

            if result.returncode != 0:
                return f"Error creating project: {result.stderr}"

            self.current_project = project_path
            return f"Created new project at {project_path}"

        except Exception as e:
            return f"Failed to create project: {str(e)}"

    async def add_dependency(self, name: str, version: str | None = None) -> str:
        """Add a dependency to Cargo.toml"""
        if not self.current_project:
            return "No active project"

        try:
            cargo_toml = self.current_project / "Cargo.toml"
            if not cargo_toml.exists():
                return "Cargo.toml not found"

            cmd = ['cargo', 'add', name]
            if version:
                cmd.extend(['--vers', version])

            result = subprocess.run(
                cmd,
                cwd=self.current_project,
                capture_output=True,
                text=True,check=True
            )

            return result.stdout if result.returncode == 0 else f"Error: {result.stderr}"

        except Exception as e:
            return f"Failed to add dependency: {str(e)}"

    async def build(self, release: bool = False) -> str:
        """Build the project"""
        if not self.current_project:
            return "No active project"

        try:
            cmd = ['cargo', 'build']
            if release:
                cmd.append('--release')

            result = subprocess.run(
                cmd,
                cwd=self.current_project,
                capture_output=True,
                text=True
            )

            return result.stdout if result.returncode == 0 else f"Build error: {result.stderr}"

        except Exception as e:
            return f"Build failed: {str(e)}"

    async def test(self) -> str:
        """Run project tests"""
        if not self.current_project:
            return "No active project"

        try:
            result = subprocess.run(
                ['cargo', 'test'],
                cwd=self.current_project,
                capture_output=True,
                text=True, check=True
            )

            return result.stdout if result.returncode == 0 else f"Test error: {result.stderr}"

        except Exception as e:
            return f"Tests failed: {str(e)}"

    async def run_code(self, code: str) -> str:
        """Run Rust code"""
        if not self.current_project:
            return "No active project"

        try:
            # Write code to main.rs
            main_rs = self.current_project / "src" / "main.rs"
            with open(main_rs, 'w') as f:
                f.write(code)

            # Build and run
            build_result = subprocess.run(
                ['cargo', 'build'],
                cwd=self.current_project,
                capture_output=True,
                text=True
            )

            if build_result.returncode != 0:
                return f"Compilation error: {build_result.stderr}"

            run_result = subprocess.run(
                ['cargo', 'run'],
                cwd=self.current_project,
                capture_output=True,
                text=True
            )

            self._execution_count += 1
            output = {
                'code': code,
                'stdout': run_result.stdout,
                'stderr': run_result.stderr,
                'result': run_result.returncode == 0
            }
            self.output_history[self._execution_count] = output

            return run_result.stdout if run_result.returncode == 0 else f"Runtime error: {run_result.stderr}"

        except Exception as e:
            return f"Execution failed: {str(e)}"

    async def modify_code(self, code: str, object_name: str, file: str = "src/main.rs") -> str:
        """Modify existing Rust code"""
        if not self.current_project:
            return "No active project"

        try:
            file_path = self.current_project / file
            if not file_path.exists():
                return f"File {file} not found"

            with open(file_path) as f:
                content = f.read()

            # Handle function modification
            if object_name.endswith("()"):
                func_name = object_name[:-2]
                # Find and replace function definition
                pattern = f"fn {func_name}.*?}}(?=\n|$)"
                updated_content = re.sub(pattern, code.strip(), content, flags=re.DOTALL)
            else:
                # Handle other modifications (structs, constants, etc.)
                pattern = f"{object_name}.*?(?=\n|$)"
                updated_content = re.sub(pattern, code.strip(), content)

            with open(file_path, 'w') as f:
                f.write(updated_content)

            return f"Modified {object_name} in {file}"

        except Exception as e:
            return f"Modification failed: {str(e)}"

    def save_session(self, name: str):
        """Save current session state"""
        session_file = self._session_dir / f"{name}.json"
        state = {
            'output_history': self.output_history,
            'current_project': str(self.current_project) if self.current_project else None
        }

        with open(session_file, 'w') as f:
            json.dump(state, f)

    def load_session(self, name: str):
        """Load saved session state"""
        session_file = self._session_dir / f"{name}.json"
        if session_file.exists():
            with open(session_file) as f:
                state = json.load(f)
                self.output_history = state['output_history']
                self.current_project = Path(state['current_project']) if state['current_project'] else None

__init__(session_dir=None, auto_remove=True)

Initialize the Rust/Cargo interface

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

def __init__(self, session_dir=None, auto_remove=True):
    """Initialize the Rust/Cargo interface"""
    self.auto_remove = auto_remove
    self._session_dir = session_dir or Path.home() / '.cargo_sessions'
    self._session_dir.mkdir(exist_ok=True)
    self.vfs = VirtualFileSystem(self._session_dir / 'virtual_fs')
    self.output_history = {}
    self._execution_count = 0
    self.current_project = None

add_dependency(name, version=None) async

Add a dependency to Cargo.toml

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

async def add_dependency(self, name: str, version: str | None = None) -> str:
    """Add a dependency to Cargo.toml"""
    if not self.current_project:
        return "No active project"

    try:
        cargo_toml = self.current_project / "Cargo.toml"
        if not cargo_toml.exists():
            return "Cargo.toml not found"

        cmd = ['cargo', 'add', name]
        if version:
            cmd.extend(['--vers', version])

        result = subprocess.run(
            cmd,
            cwd=self.current_project,
            capture_output=True,
            text=True,check=True
        )

        return result.stdout if result.returncode == 0 else f"Error: {result.stderr}"

    except Exception as e:
        return f"Failed to add dependency: {str(e)}"

build(release=False) async

Build the project

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

async def build(self, release: bool = False) -> str:
    """Build the project"""
    if not self.current_project:
        return "No active project"

    try:
        cmd = ['cargo', 'build']
        if release:
            cmd.append('--release')

        result = subprocess.run(
            cmd,
            cwd=self.current_project,
            capture_output=True,
            text=True
        )

        return result.stdout if result.returncode == 0 else f"Build error: {result.stderr}"

    except Exception as e:
        return f"Build failed: {str(e)}"

load_session(name)

Load saved session state

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

def load_session(self, name: str):
    """Load saved session state"""
    session_file = self._session_dir / f"{name}.json"
    if session_file.exists():
        with open(session_file) as f:
            state = json.load(f)
            self.output_history = state['output_history']
            self.current_project = Path(state['current_project']) if state['current_project'] else None

modify_code(code, object_name, file='src/main.rs') async

Modify existing Rust code

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

async def modify_code(self, code: str, object_name: str, file: str = "src/main.rs") -> str:
    """Modify existing Rust code"""
    if not self.current_project:
        return "No active project"

    try:
        file_path = self.current_project / file
        if not file_path.exists():
            return f"File {file} not found"

        with open(file_path) as f:
            content = f.read()

        # Handle function modification
        if object_name.endswith("()"):
            func_name = object_name[:-2]
            # Find and replace function definition
            pattern = f"fn {func_name}.*?}}(?=\n|$)"
            updated_content = re.sub(pattern, code.strip(), content, flags=re.DOTALL)
        else:
            # Handle other modifications (structs, constants, etc.)
            pattern = f"{object_name}.*?(?=\n|$)"
            updated_content = re.sub(pattern, code.strip(), content)

        with open(file_path, 'w') as f:
            f.write(updated_content)

        return f"Modified {object_name} in {file}"

    except Exception as e:
        return f"Modification failed: {str(e)}"

reset()

Reset the interface state

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

def reset(self):
    """Reset the interface state"""
    if self.auto_remove and self.current_project:
        shutil.rmtree(self.current_project, ignore_errors=True)
    self.output_history.clear()
    self._execution_count = 0
    self.current_project = None

run_code(code) async

Run Rust code

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

async def run_code(self, code: str) -> str:
    """Run Rust code"""
    if not self.current_project:
        return "No active project"

    try:
        # Write code to main.rs
        main_rs = self.current_project / "src" / "main.rs"
        with open(main_rs, 'w') as f:
            f.write(code)

        # Build and run
        build_result = subprocess.run(
            ['cargo', 'build'],
            cwd=self.current_project,
            capture_output=True,
            text=True
        )

        if build_result.returncode != 0:
            return f"Compilation error: {build_result.stderr}"

        run_result = subprocess.run(
            ['cargo', 'run'],
            cwd=self.current_project,
            capture_output=True,
            text=True
        )

        self._execution_count += 1
        output = {
            'code': code,
            'stdout': run_result.stdout,
            'stderr': run_result.stderr,
            'result': run_result.returncode == 0
        }
        self.output_history[self._execution_count] = output

        return run_result.stdout if run_result.returncode == 0 else f"Runtime error: {run_result.stderr}"

    except Exception as e:
        return f"Execution failed: {str(e)}"

save_session(name)

Save current session state

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

def save_session(self, name: str):
    """Save current session state"""
    session_file = self._session_dir / f"{name}.json"
    state = {
        'output_history': self.output_history,
        'current_project': str(self.current_project) if self.current_project else None
    }

    with open(session_file, 'w') as f:
        json.dump(state, f)

setup_project(name) async

Set up a new Cargo project

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

async def setup_project(self, name: str) -> str:
    """Set up a new Cargo project"""
    try:
        project_path = self.vfs.base_dir / name
        if project_path.exists():
            shutil.rmtree(project_path)

        result = subprocess.run(
            ['cargo', 'new', str(project_path)],
            capture_output=True,
            text=True, check=True
        )

        if result.returncode != 0:
            return f"Error creating project: {result.stderr}"

        self.current_project = project_path
        return f"Created new project at {project_path}"

    except Exception as e:
        return f"Failed to create project: {str(e)}"

test() async

Run project tests

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

async def test(self) -> str:
    """Run project tests"""
    if not self.current_project:
        return "No active project"

    try:
        result = subprocess.run(
            ['cargo', 'test'],
            cwd=self.current_project,
            capture_output=True,
            text=True, check=True
        )

        return result.stdout if result.returncode == 0 else f"Test error: {result.stderr}"

    except Exception as e:
        return f"Tests failed: {str(e)}"

MockIPython

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

class MockIPython:
    def __init__(self, _session_dir=None, auto_remove=True):
        self.auto_remove = auto_remove
        self.output_history = {}
        self._execution_count = 0
        self._session_dir = _session_dir or Path(get_app().appdata) / '.pipeline_sessions'
        self._session_dir.mkdir(exist_ok=True)
        self.vfs = VirtualFileSystem(self._session_dir / 'virtual_fs')
        self._venv_path = self._session_dir / 'venv'
        self.user_ns: dict[str, Any] = {}
        # import nest_asyncio
        # nest_asyncio.apply()
        # Set up virtual environment if it doesn't exist
        with Spinner("Starting virtual environment"):
            self._setup_venv()
        self.reset()

    def _virtual_open(self, filepath, mode='r', *args, **kwargs):
        """Custom open function that uses virtual filesystem and makes files available for import"""
        try:
            abs_path = self.vfs._resolve_path(filepath)
        except ValueError:
            # If path resolution fails, try to resolve relative to current working directory
            abs_path = self.vfs.base_dir / filepath

        if 'w' in mode or 'a' in mode:
            # Ensure parent directory exists
            abs_path.parent.mkdir(parents=True, exist_ok=True)

        # Use actual filesystem but track in virtual fs
        real_file = open(abs_path, mode, *args, **kwargs)

        if 'r' in mode:
            # Track file content in virtual filesystem when reading
            rel_path = str(abs_path.relative_to(self.vfs.base_dir))
            if rel_path not in self.vfs.virtual_files:
                try:
                    content = real_file.read()
                    self.vfs.virtual_files[rel_path] = content
                    real_file.seek(0)
                except UnicodeDecodeError:
                    # Handle binary files
                    pass

        return real_file

    def _setup_venv(self):
        """Create virtual environment if it doesn't exist"""
        if not self._venv_path.exists():
            try:
                subprocess.run([sys.executable, "-m", "venv", str(self._venv_path)], check=True)
            except subprocess.CalledProcessError as e:
                raise RuntimeError(f"Failed to create virtual environment: {str(e)}")

    def _virtual_open(self, filepath, mode='r', *args, **kwargs):
        """Custom open function that uses virtual filesystem"""
        abs_path = self.vfs._resolve_path(filepath)

        if 'w' in mode or 'a' in mode:
            # Ensure parent directory exists
            abs_path.parent.mkdir(parents=True, exist_ok=True)

        # Use actual filesystem but track in virtual fs
        real_file = open(abs_path, mode, *args, **kwargs)

        if 'r' in mode:
            # Track file content in virtual filesystem when reading
            rel_path = str(abs_path.relative_to(self.vfs.base_dir))
            if rel_path not in self.vfs.virtual_files:
                try:
                    self.vfs.virtual_files[rel_path] = real_file.read()
                    real_file.seek(0)
                except UnicodeDecodeError:
                    # Handle binary files
                    pass

        return real_file

    def reset(self):
        """Reset the interpreter state"""
        self.user_ns = {
            '__name__': '__main__',
            '__builtins__': __builtins__,
            'toolboxv2': toolboxv2,
            '__file__': None,
            '__path__': [str(self.vfs.current_dir)],
            'auto_install': auto_install,
            'app': get_app(),
            'modify_code': self.modify_code,
            'open': self._virtual_open,
        }
        self.output_history.clear()
        self._execution_count = 0
        if self.auto_remove:
            shutil.rmtree(self.vfs.base_dir, ignore_errors=True)

    def get_namespace(self) -> dict[str, Any]:
        """Get current namespace"""
        return self.user_ns.copy()

    def update_namespace(self, variables: dict[str, Any]):
        """Update namespace with new variables"""
        self.user_ns.update(variables)

    @staticmethod
    def _parse_code(code: str) -> tuple[Any, Any | None, bool, bool]:
        """Parse code and handle top-level await"""
        code_ = ""
        for line in code.split('\n'):
            if line.strip().startswith('#'):
                continue
            if line.strip().startswith('asyncio.run('):
                line = (' ' *(len(line) - len(line.strip()))) + 'await ' + line.strip()[len('asyncio.run('):-1]
            code_ += line + '\n'
        try:
            tree = ast.parse(code)
            # Add parent references
            ParentNodeTransformer().visit(tree)

            # Detect async features
            detector = AsyncCodeDetector()
            detector.visit(tree)

            if detector.has_top_level_await:
                # Wrap code in async function
                wrapped_code = "async def __wrapper():\n"
                wrapped_code += "    global result\n"  # Allow writing to global scope
                wrapped_code += "    result = None\n"
                # add try:
                wrapped_code +="    try:\n"
                # Indent the original code
                wrapped_code += "\n".join(f"        {line}" for line in code.splitlines())
                # Add return statement for last expression
                wrapped_code +="\n    except Exception as e:\n"
                wrapped_code +="        import traceback\n"
                wrapped_code +="        print(traceback.format_exc())\n"
                wrapped_code +="        raise e\n"
                if isinstance(tree.body[-1], ast.Expr):
                    wrapped_code += "\n    return result"

                # Parse and compile wrapped code
                wrapped_tree = ast.parse(wrapped_code)
                return (
                    compile(wrapped_tree, '<exec>', 'exec'),
                    None,
                    True,
                    True
                )

            # Handle regular code
            if isinstance(tree.body[-1], ast.Expr):
                exec_code = ast.Module(
                    body=tree.body[:-1],
                    type_ignores=[]
                )
                eval_code = ast.Expression(
                    body=tree.body[-1].value
                )
                return (
                    compile(exec_code, '<exec>', 'exec'),
                    compile(eval_code, '<eval>', 'eval'),
                    detector.has_async,
                    False
                )

            return (
                compile(tree, '<exec>', 'exec'),
                None,
                detector.has_async,
                False
            )

        except SyntaxError as e:
            lines = code.splitlines()
            if e.lineno and e.lineno <= len(lines):
                line = lines[e.lineno - 1]
                arrow = ' ' * (e.offset - 1) + '^' if e.offset else ''
                error_msg = (
                    f"Syntax error at line {e.lineno}:\n"
                    f"{line}\n"
                    f"{arrow}\n"
                    f"{e.msg}"
                )
            else:
                error_msg = str(e)

            error_msg += traceback.format_exc()

            raise SyntaxError(error_msg) from e

    async def run_cell(self, code: str, live_output: bool = True) -> Any:
        """Async version of run_cell that handles both sync and async code"""
        result = None
        error = None
        tb = None
        original_dir = os.getcwd()

        if live_output:
            stdout_buffer = io.StringIO()
            stderr_buffer = io.StringIO()
            stdout = TeeStream(sys.__stdout__, stdout_buffer)
            stderr = TeeStream(sys.__stderr__, stderr_buffer)
        else:
            stdout = io.StringIO()
            stderr = io.StringIO()

        try:
            # Check if a file is already specified
            original_file = self.user_ns.get('__file__')
            if original_file is None:
                # Create temp file if no file specified
                temp_file = self.vfs.write_file(
                    f'src/temp/_temp_{self._execution_count}.py',
                    code
                )
                # work_ns = self.user_ns.copy()
                self.user_ns['__file__'] = str(temp_file)
            else:
                # Use existing file
                temp_file = Path(original_file)
                # Write code to the existing file
                self.vfs.write_file(temp_file, code)
                #work_ns = self.user_ns.copy()

            self.user_ns['__builtins__'] = __builtins__
            with VirtualEnvContext(self._venv_path) as python_exec:
                try:
                    exec_code, eval_code, is_async, has_top_level_await = self._parse_code(
                        code.encode('utf-8', errors='replace').decode('utf-8')
                    )
                    if exec_code is None:
                        return "No executable code"
                    os.makedirs(str(temp_file.parent.absolute()), exist_ok=True)
                    os.chdir(str(temp_file.parent.absolute()))
                    self.user_ns['PYTHON_EXEC'] = python_exec

                    with redirect_stdout(stdout), redirect_stderr(stderr):
                        if has_top_level_await:
                            try:
                                # Execute wrapped code and await it
                                exec(exec_code, self.user_ns)
                                result = self.user_ns['__wrapper']()
                                if asyncio.iscoroutine(result):
                                    result = await result
                            finally:
                                self.user_ns.pop('__wrapper', None)
                        elif is_async:
                            # Execute async code
                            exec(exec_code, self.user_ns)
                            if eval_code:
                                result = eval(eval_code, self.user_ns)
                                if asyncio.iscoroutine(result):
                                    result = await result
                        else:
                            # Execute sync code
                            exec(exec_code, self.user_ns)
                            if eval_code:
                                result = eval(eval_code, self.user_ns)

                        if result is not None:
                            self.user_ns['_'] = result
                except KeyboardInterrupt:
                    print("Stop execution manuel!")

                except Exception as e:
                    error = str(e)
                    tb = traceback.format_exc()
                    if live_output:
                        sys.__stderr__.write(f"{error}\n{tb}")
                    stderr.write(f"{error}\n{tb}")

                finally:
                    os.chdir(original_dir)
                    self._execution_count += 1
                    # self.user_ns = work_ns.copy()
                    if live_output:
                        stdout_value = stdout_buffer.getvalue()
                        stderr_value = stderr_buffer.getvalue()
                    else:
                        stdout_value = stdout.getvalue()
                        stderr_value = stderr.getvalue()

                    output = {
                        'code': code,
                        'stdout': stdout_value,
                        'stderr': stderr_value,
                        'result': result if result else "stdout"
                    }
                    self.output_history[self._execution_count] = output

        except Exception as e:
            error_msg = f"Error executing code: {str(e)}\n{traceback.format_exc()}"
            if live_output:
                sys.__stderr__.write(error_msg)
            return error_msg

        if not result:
            result = ""
        if output['stdout']:
            result = f"{result}\nstdout:{output['stdout']}"
        if output['stderr']:
            result = f"{result}\nstderr:{output['stderr']}"

        if self.auto_remove and original_file is None:
            # Only remove temp files, not user-specified files
            self.vfs.delete_file(temp_file)

        return result

    async def modify_code(self, code: str = None, object_name: str = None, file: str = None) -> str:
        '''
        Modify existing code in memory (user namespace) and optionally in the corresponding file.

        This method updates variables, functions, or methods in the current Python session and can
        also update the corresponding source file if specified.

        Args:
            code: New value or implementation for the object
            object_name: Name of the object to modify (variable, function, or method)
            file: Path to the file to update (if None, only updates in memory)

        Returns:
            String describing the modification result

        Examples:

        # 1. Update a variable in memory
        await ipython.modify_code(code="5", object_name="x")

    # 2. Change a method implementation
    await ipython.modify_code(
        code='"""def sound(self):\n        return "Woof""""',
        object_name="Dog.sound"
    )

    # 3. Modify a function
    await ipython.modify_code(
        code='"""def calculate_age():\n    return 25"""',
        object_name="calculate_age"
    )

    # 4. Update variable in memory and file
    await ipython.modify_code(
        code="100",
        object_name="MAX_SIZE",
        file="config.py"
    )

    # 5. Modifying an attribute in __init__
    await ipython.modify_code(
        code='"""def __init__(self):\n        self.name = "Buddy""""',
        object_name="Dog.__init__"
    )
        '''
        try:
            if not object_name:
                raise ValueError("Object name must be specified")
            if code is None:
                raise ValueError("New code or value must be provided")

            # Process object name (handle methods with parentheses)
            clean_object_name = object_name.replace("()", "")

            # Step 1: Update in memory (user namespace)
            result_message = []

            # Handle different types of objects
            if "." in clean_object_name:
                # For methods or class attributes
                parts = clean_object_name.split(".")
                base_obj_name = parts[0]
                attr_name = parts[1]

                if base_obj_name not in self.user_ns:
                    raise ValueError(f"Object '{base_obj_name}' not found in namespace")

                base_obj = self.user_ns[base_obj_name]

                # Handle method definitions which are passed as docstrings
                if code.split('\n'):
                    method_code = code

                    # Parse the method code to extract its body
                    method_ast = ast.parse(method_code).body[0]
                    method_name = method_ast.name

                    # Create a new function object from the code
                    method_locals = {}
                    exec(
                        f"def _temp_func{signature(getattr(base_obj.__class__, attr_name, None))}: {method_ast.body[0].value.s}",
                        globals(), method_locals)
                    new_method = method_locals['_temp_func']

                    # Set the method on the class
                    setattr(base_obj.__class__, attr_name, new_method)
                    result_message.append(f"Updated method '{clean_object_name}' in memory")
                else:
                    # For simple attributes
                    setattr(base_obj, attr_name, eval(code, self.user_ns))
                    result_message.append(f"Updated attribute '{clean_object_name}' in memory")
            else:
                # For variables and functions
                if code.startswith('"""') and code.endswith('"""'):
                    # Handle function definitions
                    func_code = code.strip('"""')
                    func_ast = ast.parse(func_code).body[0]
                    func_name = func_ast.name

                    # Create a new function object from the code
                    func_locals = {}
                    exec(f"{func_code}", globals(), func_locals)
                    self.user_ns[clean_object_name] = func_locals[func_name]
                    result_message.append(f"Updated function '{clean_object_name}' in memory")
                else:
                    # Simple variable assignment
                    self.user_ns[clean_object_name] = eval(code, self.user_ns)
                    result_message.append(f"Updated variable '{clean_object_name}' in memory")

            # Step 2: Update in file if specified
            if file is not None:
                file_path = self.vfs._resolve_path(file)

                if not file_path.exists():
                    self.user_ns['__file__'] = str(file_path)
                    return await self.run_cell(code)

                # Read original content
                original_content = self.vfs.read_file(file_path)
                updated_content = original_content

                # Handle different object types for file updates
                if "." in clean_object_name:
                    # For methods
                    parts = clean_object_name.split(".")
                    class_name = parts[0]
                    method_name = parts[1]

                    if code.startswith('"""') and code.endswith('"""'):
                        method_code = code.strip('"""')

                        # Use ast to parse the file and find the method to replace
                        file_ast = ast.parse(original_content)
                        for node in ast.walk(file_ast):
                            if isinstance(node, ast.ClassDef) and node.name == class_name:
                                for method in node.body:
                                    if isinstance(method, ast.FunctionDef) and method.name == method_name:
                                        # Find the method in the source code
                                        method_pattern = fr"def {method_name}.*?:(.*?)(?=\n    \w|\n\w|\Z)"
                                        method_match = re.search(method_pattern, original_content, re.DOTALL)

                                        if method_match:
                                            indentation = re.match(r"^(\s*)", method_match.group(0)).group(1)
                                            method_indented = textwrap.indent(method_code, indentation)
                                            updated_content = original_content.replace(
                                                method_match.group(0),
                                                method_indented
                                            )
                                            self.vfs.write_file(file_path, updated_content)
                                            result_message.append(
                                                f"Updated method '{clean_object_name}' in file '{file}'")
                else:
                    # For variables and functions
                    if code.startswith('"""') and code.endswith('"""'):
                        # Handle function updates
                        func_code = code.strip('"""')
                        func_pattern = fr"def {clean_object_name}.*?:(.*?)(?=\n\w|\Z)"
                        func_match = re.search(func_pattern, original_content, re.DOTALL)

                        if func_match:
                            indentation = re.match(r"^(\s*)", func_match.group(0)).group(1)
                            func_indented = textwrap.indent(func_code, indentation)
                            updated_content = original_content.replace(
                                func_match.group(0),
                                func_indented
                            )
                            self.vfs.write_file(file_path, updated_content)
                            result_message.append(f"Updated function '{clean_object_name}' in file '{file}'")
                    else:
                        # Handle variable updates
                        var_pattern = fr"{clean_object_name}\s*=.*"
                        var_replacement = f"{clean_object_name} = {code}"
                        updated_content = re.sub(var_pattern, var_replacement, original_content)

                        if updated_content != original_content:
                            self.vfs.write_file(file_path, updated_content)
                            result_message.append(f"Updated variable '{clean_object_name}' in file '{file}'")
                        else:
                            result_message.append(f"Could not find variable '{clean_object_name}' in file '{file}'")

            return "\n".join(result_message)

        except Exception as e:
            return f"Error during code modification: {str(e)}\n{traceback.format_exc()}"


    def save_session(self, name: str):
        """Save session with UTF-8 encoding"""
        session_file = self._session_dir / f"{name}.pkl"
        user_ns = self.user_ns.copy()
        output_history = self.output_history.copy()

        # Ensure all strings are properly encoded
        for key, value in user_ns.items():
            try:
                if isinstance(value, str):
                    value = value.encode('utf-8').decode('utf-8')
                pickle.dumps(value)
            except Exception:
                user_ns[key] = f"not serializable: {str(value)}"

        for key, value in output_history.items():
            try:
                if isinstance(value, dict):
                    for k, v in value.items():
                        if isinstance(v, str):
                            value[k] = v.encode('utf-8').decode('utf-8')
                pickle.dumps(value)
            except Exception:
                output_history[key] = f"not serializable: {str(value)}"


        session_data = {
            'user_ns': user_ns,
            'output_history': output_history,

        }

        with open(session_file, 'wb') as f:
            pickle.dump(session_data, f)

        # Save VFS state with UTF-8 encoding
        vfs_state_file = self._session_dir / f"{name}_vfs.json"
        with open(vfs_state_file, 'w', encoding='utf-8') as f:
            json.dump(self.vfs.virtual_files, f, ensure_ascii=False)

    def load_session(self, name: str):
        """Load session with UTF-8 encoding"""
        session_file = self._session_dir / f"{name}.pkl"
        if session_file.exists():
            with open(session_file, 'rb') as f:
                session_data = pickle.load(f)
                # self.user_ns.update(session_data['user_ns'])
                self.output_history.update(session_data['output_history'])

        # Load VFS state with UTF-8 encoding
        vfs_state_file = self._session_dir / f"{name}_vfs.json"
        if vfs_state_file.exists():
            with open(vfs_state_file, encoding='utf-8') as f:
                self.vfs.virtual_files = json.load(f)

    def __str__(self):
        """String representation of current session"""
        output = []
        for count, data in self.output_history.items():
            output.append(f"In [{count}]: {data['code']}")
            if data['stdout']:
                output.append(data['stdout'])
            if data['stderr']:
                output.append(f"Error: {data['stderr']}")
            if data['result'] is not None:
                output.append(f"Out[{count}]: {data['result']}")
        return "\n".join(output)

    def set_base_directory(self, path: str) -> str:
        """
        Set the base directory for the virtual file system and add it to sys.path for imports.

        Args:
            path: New base directory path

        Returns:
            Success message
        """
        try:
            new_path = Path(path) if isinstance(path, str) else path
            new_path.mkdir(parents=True, exist_ok=True)

            # Remove old base directory from sys.path if it exists
            old_base_str = str(self.vfs.base_dir)
            if old_base_str in sys.path:
                sys.path.remove(old_base_str)

            # Update VFS base directory
            self.vfs.base_dir = new_path
            self.vfs.current_dir = new_path

            # Add new base directory to sys.path for imports
            new_base_str = str(new_path)
            if new_base_str not in sys.path:
                sys.path.insert(0, new_base_str)

            # Update user namespace paths
            self.user_ns['__path__'] = [new_base_str]

            return f"Base directory set to: {new_path} (added to sys.path)"

        except Exception as e:
            return f"Set base directory error: {str(e)}"

__str__()

String representation of current session

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

def __str__(self):
    """String representation of current session"""
    output = []
    for count, data in self.output_history.items():
        output.append(f"In [{count}]: {data['code']}")
        if data['stdout']:
            output.append(data['stdout'])
        if data['stderr']:
            output.append(f"Error: {data['stderr']}")
        if data['result'] is not None:
            output.append(f"Out[{count}]: {data['result']}")
    return "\n".join(output)

get_namespace()

Get current namespace

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

def get_namespace(self) -> dict[str, Any]:
    """Get current namespace"""
    return self.user_ns.copy()

load_session(name)

Load session with UTF-8 encoding

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

def load_session(self, name: str):
    """Load session with UTF-8 encoding"""
    session_file = self._session_dir / f"{name}.pkl"
    if session_file.exists():
        with open(session_file, 'rb') as f:
            session_data = pickle.load(f)
            # self.user_ns.update(session_data['user_ns'])
            self.output_history.update(session_data['output_history'])

    # Load VFS state with UTF-8 encoding
    vfs_state_file = self._session_dir / f"{name}_vfs.json"
    if vfs_state_file.exists():
        with open(vfs_state_file, encoding='utf-8') as f:
            self.vfs.virtual_files = json.load(f)

modify_code(code=None, object_name=None, file=None) async

Modify existing code in memory (user namespace) and optionally in the corresponding file.

This method updates variables, functions, or methods in the current Python session and can
also update the corresponding source file if specified.

Args:
    code: New value or implementation for the object
    object_name: Name of the object to modify (variable, function, or method)
    file: Path to the file to update (if None, only updates in memory)

Returns:
    String describing the modification result

Examples:

# 1. Update a variable in memory
await ipython.modify_code(code="5", object_name="x")

2. Change a method implementation

await ipython.modify_code( code='"""def sound(self): return "Woof""""', object_name="Dog.sound" )

3. Modify a function

await ipython.modify_code( code='"""def calculate_age(): return 25"""', object_name="calculate_age" )

4. Update variable in memory and file

await ipython.modify_code( code="100", object_name="MAX_SIZE", file="config.py" )

5. Modifying an attribute in init

await ipython.modify_code( code='"""def init(self): self.name = "Buddy""""', object_name="Dog.init" )

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

async def modify_code(self, code: str = None, object_name: str = None, file: str = None) -> str:
    '''
    Modify existing code in memory (user namespace) and optionally in the corresponding file.

    This method updates variables, functions, or methods in the current Python session and can
    also update the corresponding source file if specified.

    Args:
        code: New value or implementation for the object
        object_name: Name of the object to modify (variable, function, or method)
        file: Path to the file to update (if None, only updates in memory)

    Returns:
        String describing the modification result

    Examples:

    # 1. Update a variable in memory
    await ipython.modify_code(code="5", object_name="x")

# 2. Change a method implementation
await ipython.modify_code(
    code='"""def sound(self):\n        return "Woof""""',
    object_name="Dog.sound"
)

# 3. Modify a function
await ipython.modify_code(
    code='"""def calculate_age():\n    return 25"""',
    object_name="calculate_age"
)

# 4. Update variable in memory and file
await ipython.modify_code(
    code="100",
    object_name="MAX_SIZE",
    file="config.py"
)

# 5. Modifying an attribute in __init__
await ipython.modify_code(
    code='"""def __init__(self):\n        self.name = "Buddy""""',
    object_name="Dog.__init__"
)
    '''
    try:
        if not object_name:
            raise ValueError("Object name must be specified")
        if code is None:
            raise ValueError("New code or value must be provided")

        # Process object name (handle methods with parentheses)
        clean_object_name = object_name.replace("()", "")

        # Step 1: Update in memory (user namespace)
        result_message = []

        # Handle different types of objects
        if "." in clean_object_name:
            # For methods or class attributes
            parts = clean_object_name.split(".")
            base_obj_name = parts[0]
            attr_name = parts[1]

            if base_obj_name not in self.user_ns:
                raise ValueError(f"Object '{base_obj_name}' not found in namespace")

            base_obj = self.user_ns[base_obj_name]

            # Handle method definitions which are passed as docstrings
            if code.split('\n'):
                method_code = code

                # Parse the method code to extract its body
                method_ast = ast.parse(method_code).body[0]
                method_name = method_ast.name

                # Create a new function object from the code
                method_locals = {}
                exec(
                    f"def _temp_func{signature(getattr(base_obj.__class__, attr_name, None))}: {method_ast.body[0].value.s}",
                    globals(), method_locals)
                new_method = method_locals['_temp_func']

                # Set the method on the class
                setattr(base_obj.__class__, attr_name, new_method)
                result_message.append(f"Updated method '{clean_object_name}' in memory")
            else:
                # For simple attributes
                setattr(base_obj, attr_name, eval(code, self.user_ns))
                result_message.append(f"Updated attribute '{clean_object_name}' in memory")
        else:
            # For variables and functions
            if code.startswith('"""') and code.endswith('"""'):
                # Handle function definitions
                func_code = code.strip('"""')
                func_ast = ast.parse(func_code).body[0]
                func_name = func_ast.name

                # Create a new function object from the code
                func_locals = {}
                exec(f"{func_code}", globals(), func_locals)
                self.user_ns[clean_object_name] = func_locals[func_name]
                result_message.append(f"Updated function '{clean_object_name}' in memory")
            else:
                # Simple variable assignment
                self.user_ns[clean_object_name] = eval(code, self.user_ns)
                result_message.append(f"Updated variable '{clean_object_name}' in memory")

        # Step 2: Update in file if specified
        if file is not None:
            file_path = self.vfs._resolve_path(file)

            if not file_path.exists():
                self.user_ns['__file__'] = str(file_path)
                return await self.run_cell(code)

            # Read original content
            original_content = self.vfs.read_file(file_path)
            updated_content = original_content

            # Handle different object types for file updates
            if "." in clean_object_name:
                # For methods
                parts = clean_object_name.split(".")
                class_name = parts[0]
                method_name = parts[1]

                if code.startswith('"""') and code.endswith('"""'):
                    method_code = code.strip('"""')

                    # Use ast to parse the file and find the method to replace
                    file_ast = ast.parse(original_content)
                    for node in ast.walk(file_ast):
                        if isinstance(node, ast.ClassDef) and node.name == class_name:
                            for method in node.body:
                                if isinstance(method, ast.FunctionDef) and method.name == method_name:
                                    # Find the method in the source code
                                    method_pattern = fr"def {method_name}.*?:(.*?)(?=\n    \w|\n\w|\Z)"
                                    method_match = re.search(method_pattern, original_content, re.DOTALL)

                                    if method_match:
                                        indentation = re.match(r"^(\s*)", method_match.group(0)).group(1)
                                        method_indented = textwrap.indent(method_code, indentation)
                                        updated_content = original_content.replace(
                                            method_match.group(0),
                                            method_indented
                                        )
                                        self.vfs.write_file(file_path, updated_content)
                                        result_message.append(
                                            f"Updated method '{clean_object_name}' in file '{file}'")
            else:
                # For variables and functions
                if code.startswith('"""') and code.endswith('"""'):
                    # Handle function updates
                    func_code = code.strip('"""')
                    func_pattern = fr"def {clean_object_name}.*?:(.*?)(?=\n\w|\Z)"
                    func_match = re.search(func_pattern, original_content, re.DOTALL)

                    if func_match:
                        indentation = re.match(r"^(\s*)", func_match.group(0)).group(1)
                        func_indented = textwrap.indent(func_code, indentation)
                        updated_content = original_content.replace(
                            func_match.group(0),
                            func_indented
                        )
                        self.vfs.write_file(file_path, updated_content)
                        result_message.append(f"Updated function '{clean_object_name}' in file '{file}'")
                else:
                    # Handle variable updates
                    var_pattern = fr"{clean_object_name}\s*=.*"
                    var_replacement = f"{clean_object_name} = {code}"
                    updated_content = re.sub(var_pattern, var_replacement, original_content)

                    if updated_content != original_content:
                        self.vfs.write_file(file_path, updated_content)
                        result_message.append(f"Updated variable '{clean_object_name}' in file '{file}'")
                    else:
                        result_message.append(f"Could not find variable '{clean_object_name}' in file '{file}'")

        return "\n".join(result_message)

    except Exception as e:
        return f"Error during code modification: {str(e)}\n{traceback.format_exc()}"

reset()

Reset the interpreter state

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

def reset(self):
    """Reset the interpreter state"""
    self.user_ns = {
        '__name__': '__main__',
        '__builtins__': __builtins__,
        'toolboxv2': toolboxv2,
        '__file__': None,
        '__path__': [str(self.vfs.current_dir)],
        'auto_install': auto_install,
        'app': get_app(),
        'modify_code': self.modify_code,
        'open': self._virtual_open,
    }
    self.output_history.clear()
    self._execution_count = 0
    if self.auto_remove:
        shutil.rmtree(self.vfs.base_dir, ignore_errors=True)

run_cell(code, live_output=True) async

Async version of run_cell that handles both sync and async code

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

async def run_cell(self, code: str, live_output: bool = True) -> Any:
    """Async version of run_cell that handles both sync and async code"""
    result = None
    error = None
    tb = None
    original_dir = os.getcwd()

    if live_output:
        stdout_buffer = io.StringIO()
        stderr_buffer = io.StringIO()
        stdout = TeeStream(sys.__stdout__, stdout_buffer)
        stderr = TeeStream(sys.__stderr__, stderr_buffer)
    else:
        stdout = io.StringIO()
        stderr = io.StringIO()

    try:
        # Check if a file is already specified
        original_file = self.user_ns.get('__file__')
        if original_file is None:
            # Create temp file if no file specified
            temp_file = self.vfs.write_file(
                f'src/temp/_temp_{self._execution_count}.py',
                code
            )
            # work_ns = self.user_ns.copy()
            self.user_ns['__file__'] = str(temp_file)
        else:
            # Use existing file
            temp_file = Path(original_file)
            # Write code to the existing file
            self.vfs.write_file(temp_file, code)
            #work_ns = self.user_ns.copy()

        self.user_ns['__builtins__'] = __builtins__
        with VirtualEnvContext(self._venv_path) as python_exec:
            try:
                exec_code, eval_code, is_async, has_top_level_await = self._parse_code(
                    code.encode('utf-8', errors='replace').decode('utf-8')
                )
                if exec_code is None:
                    return "No executable code"
                os.makedirs(str(temp_file.parent.absolute()), exist_ok=True)
                os.chdir(str(temp_file.parent.absolute()))
                self.user_ns['PYTHON_EXEC'] = python_exec

                with redirect_stdout(stdout), redirect_stderr(stderr):
                    if has_top_level_await:
                        try:
                            # Execute wrapped code and await it
                            exec(exec_code, self.user_ns)
                            result = self.user_ns['__wrapper']()
                            if asyncio.iscoroutine(result):
                                result = await result
                        finally:
                            self.user_ns.pop('__wrapper', None)
                    elif is_async:
                        # Execute async code
                        exec(exec_code, self.user_ns)
                        if eval_code:
                            result = eval(eval_code, self.user_ns)
                            if asyncio.iscoroutine(result):
                                result = await result
                    else:
                        # Execute sync code
                        exec(exec_code, self.user_ns)
                        if eval_code:
                            result = eval(eval_code, self.user_ns)

                    if result is not None:
                        self.user_ns['_'] = result
            except KeyboardInterrupt:
                print("Stop execution manuel!")

            except Exception as e:
                error = str(e)
                tb = traceback.format_exc()
                if live_output:
                    sys.__stderr__.write(f"{error}\n{tb}")
                stderr.write(f"{error}\n{tb}")

            finally:
                os.chdir(original_dir)
                self._execution_count += 1
                # self.user_ns = work_ns.copy()
                if live_output:
                    stdout_value = stdout_buffer.getvalue()
                    stderr_value = stderr_buffer.getvalue()
                else:
                    stdout_value = stdout.getvalue()
                    stderr_value = stderr.getvalue()

                output = {
                    'code': code,
                    'stdout': stdout_value,
                    'stderr': stderr_value,
                    'result': result if result else "stdout"
                }
                self.output_history[self._execution_count] = output

    except Exception as e:
        error_msg = f"Error executing code: {str(e)}\n{traceback.format_exc()}"
        if live_output:
            sys.__stderr__.write(error_msg)
        return error_msg

    if not result:
        result = ""
    if output['stdout']:
        result = f"{result}\nstdout:{output['stdout']}"
    if output['stderr']:
        result = f"{result}\nstderr:{output['stderr']}"

    if self.auto_remove and original_file is None:
        # Only remove temp files, not user-specified files
        self.vfs.delete_file(temp_file)

    return result

save_session(name)

Save session with UTF-8 encoding

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

def save_session(self, name: str):
    """Save session with UTF-8 encoding"""
    session_file = self._session_dir / f"{name}.pkl"
    user_ns = self.user_ns.copy()
    output_history = self.output_history.copy()

    # Ensure all strings are properly encoded
    for key, value in user_ns.items():
        try:
            if isinstance(value, str):
                value = value.encode('utf-8').decode('utf-8')
            pickle.dumps(value)
        except Exception:
            user_ns[key] = f"not serializable: {str(value)}"

    for key, value in output_history.items():
        try:
            if isinstance(value, dict):
                for k, v in value.items():
                    if isinstance(v, str):
                        value[k] = v.encode('utf-8').decode('utf-8')
            pickle.dumps(value)
        except Exception:
            output_history[key] = f"not serializable: {str(value)}"


    session_data = {
        'user_ns': user_ns,
        'output_history': output_history,

    }

    with open(session_file, 'wb') as f:
        pickle.dump(session_data, f)

    # Save VFS state with UTF-8 encoding
    vfs_state_file = self._session_dir / f"{name}_vfs.json"
    with open(vfs_state_file, 'w', encoding='utf-8') as f:
        json.dump(self.vfs.virtual_files, f, ensure_ascii=False)

set_base_directory(path)

Set the base directory for the virtual file system and add it to sys.path for imports.

Parameters:

Name	Type	Description	Default
path	str	New base directory path	required

Returns:

Type	Description
str	Success message

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

def set_base_directory(self, path: str) -> str:
    """
    Set the base directory for the virtual file system and add it to sys.path for imports.

    Args:
        path: New base directory path

    Returns:
        Success message
    """
    try:
        new_path = Path(path) if isinstance(path, str) else path
        new_path.mkdir(parents=True, exist_ok=True)

        # Remove old base directory from sys.path if it exists
        old_base_str = str(self.vfs.base_dir)
        if old_base_str in sys.path:
            sys.path.remove(old_base_str)

        # Update VFS base directory
        self.vfs.base_dir = new_path
        self.vfs.current_dir = new_path

        # Add new base directory to sys.path for imports
        new_base_str = str(new_path)
        if new_base_str not in sys.path:
            sys.path.insert(0, new_base_str)

        # Update user namespace paths
        self.user_ns['__path__'] = [new_base_str]

        return f"Base directory set to: {new_path} (added to sys.path)"

    except Exception as e:
        return f"Set base directory error: {str(e)}"

update_namespace(variables)

Update namespace with new variables

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

def update_namespace(self, variables: dict[str, Any]):
    """Update namespace with new variables"""
    self.user_ns.update(variables)

ParentNodeTransformer

Bases: NodeTransformer

Add parent references to AST nodes

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

class ParentNodeTransformer(ast.NodeTransformer):
    """Add parent references to AST nodes"""
    def visit(self, node):
        for child in ast.iter_child_nodes(node):
            child.parent = node
        return super().visit(node)

SyncReport dataclass

Report of variables synced from namespace to pipeline

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

@dataclass
class SyncReport:
    """Report of variables synced from namespace to pipeline"""
    added: dict[str, str]
    skipped: dict[str, str]  # var_name -> reason
    errors: dict[str, str]  # var_name -> error message

    def __str__(self) -> str:
        parts = []
        if self.added:
            parts.append("Added variables:")
            for name, type_ in self.added.items():
                parts.append(f"  - {name}: {type_}")
        if self.skipped:
            parts.append("\nSkipped variables:")
            for name, reason in self.skipped.items():
                parts.append(f"  - {name}: {reason}")
        if self.errors:
            parts.append("\nErrors:")
            for name, error in self.errors.items():
                parts.append(f"  - {name}: {error}")
        return "\n".join(parts)

TeeStream

Stream that writes to both console and buffer

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

class TeeStream:
    """Stream that writes to both console and buffer"""
    def __init__(self, console_stream, buffer_stream):
        self.console_stream = console_stream
        self.buffer_stream = buffer_stream

    def write(self, data):
        self.console_stream.write(data)
        self.buffer_stream.write(data)
        self.console_stream.flush()  # Ensure immediate console output

    def flush(self):
        self.console_stream.flush()
        self.buffer_stream.flush()

ToolsInterface

Minimalistic tools interface for LLMs providing code execution, virtual file system, and browser interaction capabilities.

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

class ToolsInterface:
    """
    Minimalistic tools interface for LLMs providing code execution,
    virtual file system, and browser interaction capabilities.
    """

    def __init__(self,
                 session_dir: str | None = None,
                 auto_remove: bool = True,
                 variables: dict[str, Any] | None = None,
                 variable_manager: Any | None = None):
        """
        Initialize the tools interface.

        Args:
            session_dir: Directory for session storage
            auto_remove: Whether to auto-remove temporary files
            variables: Initial variables dictionary
            variable_manager: External variable manager instance
            web_llm: LLM model for web interactions
        """
        self._session_dir = Path(session_dir) if session_dir else Path(get_app().appdata) / '.tools_sessions'
        self._session_dir.mkdir(exist_ok=True)
        self.auto_remove = auto_remove
        self.variable_manager = variable_manager

        # Initialize Python execution environment
        self.ipython = MockIPython(self._session_dir, auto_remove=auto_remove)
        if variables:
            self.ipython.user_ns.update(variables)

        # Initialize virtual file system
        self.vfs = VirtualFileSystem(self._session_dir / 'virtual_fs')

        # Initialize Rust interface
        self.cargo = CargoRustInterface(self._session_dir, auto_remove=auto_remove)

        # Track execution state
        self._execution_history = []
        self._current_file = None

    async def execute_python(self, code: str) -> str:
        """
        Execute Python code in the virtual environment.

        Args:
            code: Python code to execute

        Returns:
            Execution result as string
        """
        try:
            result = await self.ipython.run_cell(code, live_output=False)

            # Update variable manager if available
            if self.variable_manager:
                for key, value in self.ipython.user_ns.items():
                    if not key.startswith('_') and key not in ['__name__', '__builtins__']:
                        try:
                            self.variable_manager.set(f"python.{key}", value)
                        except:
                            pass  # Ignore non-serializable variables

            self._execution_history.append(('python', code, result))
            return str(result) if result else "Execution completed"

        except Exception as e:
            error_msg = f"Python execution error: {str(e)}\n{traceback.format_exc()}"
            self._execution_history.append(('python', code, error_msg))
            return error_msg

    async def execute_rust(self, code: str) -> str:
        """
        Execute Rust code using Cargo.

        Args:
            code: Rust code to execute

        Returns:
            Execution result as string
        """
        try:
            # Setup project if needed
            if not self.cargo.current_project:
                await self.cargo.setup_project("temp_rust_project")

            result = await self.cargo.run_code(code)
            self._execution_history.append(('rust', code, result))
            return result

        except Exception as e:
            error_msg = f"Rust execution error: {str(e)}"
            self._execution_history.append(('rust', code, error_msg))
            return error_msg

    async def write_file(self, filepath: str, content: str, lines: str = "") -> str:
        """
        Write content to a file in the virtual file system.

        Args:
            filepath: Path to the file
            content: Content to write
            lines: Optional line range to write (e.g., "1-3" for lines 1 to 3)

        Returns:
            Success message
        """

        try:
            if lines:
                abs_path = self.vfs.overwrite_lines(filepath, content, lines)
            else:
                abs_path = self.vfs.write_file(filepath, content)

            # Update variable manager if available
            if self.variable_manager:
                self.variable_manager.set(f"files.{filepath.replace('/', '.')}", {
                    'path': str(abs_path),
                    'size': len(content),
                    'content_preview': content[:100] + '...' if len(content) > 100 else content
                })

            return f"File written successfully: {abs_path}"

        except Exception as e:
            return f"File write error: {str(e)}"

    async def replace_in_file(self, filepath: str, old_content: str, new_content: str, precise: bool = True) -> str:
        """
        Replace exact content in file with new content.

        Args:
            filepath: Path to the file
            old_content: Exact content to replace (empty string for insertion at start)
            new_content: Content to replace with
            precise: If True, requires exact match; if False, allows single occurrence replacement

        Returns:
            Success message or error
        """
        try:
            # Read current file content
            try:
                current_content = self.vfs.read_file(filepath)
            except:
                return f"Error: File '{filepath}' not found or cannot be read"

            # Handle insertion at start (empty old_content)
            if not old_content:
                updated_content = new_content + current_content
                self.vfs.write_file(filepath, updated_content)
                return f"Content inserted at start of '{filepath}'"

            # Check if old_content exists
            if old_content not in current_content:
                return f"Error: Old content not found in '{filepath}' use read_file to check."

            # Count occurrences
            occurrences = current_content.count(old_content)

            if precise and occurrences > 1:
                return f"Error: Found {occurrences} occurrences of old content. Use precise=False to replace first occurrence."

            # Replace content (first occurrence if multiple)
            updated_content = current_content.replace(old_content, new_content, 1)

            # Write updated content
            self.vfs.write_file(filepath, updated_content)

            return f"Successfully replaced content in '{filepath}' ({occurrences} occurrence{'s' if occurrences > 1 else ''} found, 1 replaced)"

        except Exception as e:
            return f"Replace error: {str(e)}"

    async def read_file(self, filepath: str, lines: str="") -> str:
        """
        Read content from a file in the virtual file system.

        Args:
            filepath: Path to the file
            lines: Optional line range to read (e.g., "1-3" for lines 1 to 3)

        Returns:
            File content or error message
        """
        try:
            content = self.vfs.read_file(filepath)

            if lines:
                start, end = map(int, lines.split('-'))
                content = '\n'.join(content.split('\n')[start-1:end])
            # Update variable manager if available
            if self.variable_manager:
                self.variable_manager.set("files.last_read", {
                    'path': filepath,
                    'size': len(content),
                    'content_preview': content[:200] + '...' if len(content) > 200 else content
                })

            return content

        except Exception as e:
            return f"File read error: {str(e)}"

    async def list_files(self, dirpath: str = '.') -> str:
        """
        List files in a directory.

        Args:
            dirpath: Directory path to list

        Returns:
            File listing as string
        """
        try:
            files = self.vfs.list_files(dirpath)
            listing = "\n".join(f"- {file}" for file in files)
            return f"Files in '{dirpath}':\n{listing}"

        except Exception as e:
            return f"File listing error: {str(e)}"

    async def list_directory(self, dirpath: str = '.') -> str:
        """
        List contents of a directory.

        Args:
            dirpath: Directory path to list

        Returns:
            Directory listing as string
        """
        try:
            contents = self.vfs.list_directory(dirpath)
            listing = "\n".join(f"- {item}" for item in contents)

            # Update variable manager if available
            if self.variable_manager:
                self.variable_manager.set("files.last_listing", {
                    'directory': dirpath,
                    'items': contents,
                    'count': len(contents)
                })

            return f"Directory '{dirpath}' contents:\n{listing}"

        except Exception as e:
            return f"Directory listing error: {str(e)}"


    async def create_directory(self, dirpath: str) -> str:
        """
        Create a new directory.

        Args:
            dirpath: Path of directory to create

        Returns:
            Success message
        """
        try:
            abs_path = self.vfs.create_directory(dirpath)
            return f"Directory created successfully: {abs_path}"

        except Exception as e:
            return f"Directory creation error: {str(e)}"

    def set_base_directory(self, path: str) -> str:
        """
        Set the base directory for the virtual file system.

        Args:
            path: New base directory path

        Returns:
            Success message
        """
        try:
            new_path = Path(path) if isinstance(path, str) else path
            new_path = new_path.absolute()
            print(f"New path: {new_path}")
            new_path.mkdir(parents=True, exist_ok=True)
            self.vfs.base_dir = new_path
            self.vfs.current_dir = new_path

            # Update MockIPython base directory and sys.path
            result = self.ipython.set_base_directory(path)

            return result

        except Exception as e:
            return f"Set base directory error: {str(e)}"

    async def set_current_file(self, filepath: str) -> str:
        """
        Set the current file for Python execution context.

        Args:
            filepath: Path to set as current file

        Returns:
            Success message
        """
        try:
            abs_path = self.vfs._resolve_path(filepath)
            self.ipython.user_ns['__file__'] = str(abs_path)
            self._current_file = str(abs_path)

            return f"Current file set to: {abs_path}"

        except Exception as e:
            return f"Set current file error: {str(e)}"

    async def install_package(self, package_name: str, version: str | None = None) -> str:
        """
        Install a Python package in the virtual environment.

        Args:
            package_name: Name of the package to install
            version: Optional specific version to install

        Returns:
            Installation result
        """
        try:
            code = f"""
auto_install('{package_name}'{f", version='{version}'" if version else ""})
import {package_name.split('[')[0]}  # Import base package name
print(f"Successfully imported {package_name}")
"""
            result = await self.execute_python(code)
            return result

        except Exception as e:
            return f"Package installation error: {str(e)}"

    async def get_execution_history(self) -> str:
        """
        Get the execution history.

        Returns:
            Execution history as formatted string
        """
        if not self._execution_history:
            return "No execution history available."

        history_lines = []
        for i, (lang, code, result) in enumerate(self._execution_history[-10:], 1):
            history_lines.append(f"[{i}] {lang.upper()}:")
            history_lines.append(f"    Code: {code[:100]}..." if len(code) > 100 else f"    Code: {code}")
            history_lines.append(
                f"    Result: {str(result)[:200]}..." if len(str(result)) > 200 else f"    Result: {result}")
            history_lines.append("")

        return "\n".join(history_lines)

    async def clear_session(self) -> str:
        """
        Clear the current session (variables, history, files).

        Returns:
            Success message
        """
        try:
            # Reset Python environment
            self.ipython.reset()

            # Clear execution history
            self._execution_history.clear()

            # Clear VFS if auto_remove is enabled
            if self.auto_remove:
                shutil.rmtree(self.vfs.base_dir, ignore_errors=True)
                self.vfs.base_dir.mkdir(parents=True, exist_ok=True)
                self.vfs.virtual_files.clear()

            # Reset current file
            self._current_file = None

            return "Session cleared successfully"

        except Exception as e:
            return f"Clear session error: {str(e)}"

    async def get_variables(self) -> str:
        """
        Get current variables in JSON format.

        Returns:
            Variables as JSON string
        """
        try:
            # Get Python variables
            py_vars = {}
            for key, value in self.ipython.user_ns.items():
                if not key.startswith('_') and key not in ['__name__', '__builtins__']:
                    try:
                        # Try to serialize the value
                        json.dumps(value, default=str)
                        py_vars[key] = str(value)[:200] if len(str(value)) > 200 else value
                    except:
                        py_vars[key] = f"<{type(value).__name__}>"

            result = {
                'python_variables': py_vars,
                'current_file': self._current_file,
                'vfs_base': str(self.vfs.base_dir),
                'execution_count': len(self._execution_history)
            }

            return json.dumps(result, indent=2, default=str)

        except Exception as e:
            return f"Get variables error: {str(e)}"

    def get_tools(self, name:str=None) -> list[tuple[Any, str, str]]:
        """
        Get all available tools as list of tuples (function, name, description).

        Returns:
            List of tool tuples
        """
        tools = [
            # Code execution tools
            (self.execute_python, "execute_python",
             "Execute Python code in virtual environment. all variables ar available under the python scope.\n"
             "The isaa_instance is available as isaa_instance in the python code."
             " Args: code (str) -> str"),

            # (self.execute_rust, "execute_rust",
            #  "Execute Rust code using Cargo. Args: code (str) -> str"),

            # File system tools
            (self.write_file, "write_file",
             "Write content to file in virtual filesystem. lines is a string with the line range to write (e.g., '1-3' for lines 1 to 3) Args: filepath (str), content (str), lines (str) = '' -> str"),

            (self.write_file, "create_file",
             "Write content to file in virtual filesystem.  Args: filepath (str), content (str) -> str"),

            (self.replace_in_file, "replace_in_file",
             "Replace exact content in file. Args: filepath (str), old_content (str), new_content (str), precise (bool) = True -> str"),

            (self.read_file, "read_file",
             "Read content from file in virtual filesystem. lines is a string with the line range to read (e.g., '1-3' for lines 1 to 3) Args: filepath (str), lines (str) = '' -> str"),

            (self.list_files, "list_files",
             "List files in directory. Args: dirpath (str) = '.' -> str"),

            (self.list_directory, "list_directory",
             "List directory contents. Args: dirpath (str) = '.' -> str"),

            (self.create_directory, "create_directory",
             "Create new directory. Args: dirpath (str) -> str"),

            # Configuration tools
            (self.set_base_directory, "set_base_directory",
             "Set base directory for virtual filesystem. Args: path (str) -> str"),

            (self.set_current_file, "set_current_file",
             "Set current file for Python execution context. Args: filepath (str) -> str"),

            (self.install_package, "install_package",
             "Install Python package. Args: package_name (str), version (Optional[str]) -> str"),

            # Session management tools
            (self.get_execution_history, "get_execution_history",
             "Get execution history. Args: None -> str"),

            (self.clear_session, "clear_session",
             "Clear current session. Args: None -> str"),

            (self.get_variables, "get_variables",
             "Get current variables as JSON. Args: None -> str"),
        ]
        if name is not None:
            tools = [t for t in tools if t[1] == name][0]
        return tools

    def __aenter__(self):
        return self

    async def __aexit__(self, *exe):
        await asyncio.sleep(0.01)

__init__(session_dir=None, auto_remove=True, variables=None, variable_manager=None)

Initialize the tools interface.

Parameters:

Name	Type	Description	Default
session_dir	str | None	Directory for session storage	None
auto_remove	bool	Whether to auto-remove temporary files	True
variables	dict[str, Any] | None	Initial variables dictionary	None
variable_manager	Any | None	External variable manager instance	None
web_llm		LLM model for web interactions	required

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

def __init__(self,
             session_dir: str | None = None,
             auto_remove: bool = True,
             variables: dict[str, Any] | None = None,
             variable_manager: Any | None = None):
    """
    Initialize the tools interface.

    Args:
        session_dir: Directory for session storage
        auto_remove: Whether to auto-remove temporary files
        variables: Initial variables dictionary
        variable_manager: External variable manager instance
        web_llm: LLM model for web interactions
    """
    self._session_dir = Path(session_dir) if session_dir else Path(get_app().appdata) / '.tools_sessions'
    self._session_dir.mkdir(exist_ok=True)
    self.auto_remove = auto_remove
    self.variable_manager = variable_manager

    # Initialize Python execution environment
    self.ipython = MockIPython(self._session_dir, auto_remove=auto_remove)
    if variables:
        self.ipython.user_ns.update(variables)

    # Initialize virtual file system
    self.vfs = VirtualFileSystem(self._session_dir / 'virtual_fs')

    # Initialize Rust interface
    self.cargo = CargoRustInterface(self._session_dir, auto_remove=auto_remove)

    # Track execution state
    self._execution_history = []
    self._current_file = None

clear_session() async

Clear the current session (variables, history, files).

Returns:

Type	Description
str	Success message

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

async def clear_session(self) -> str:
    """
    Clear the current session (variables, history, files).

    Returns:
        Success message
    """
    try:
        # Reset Python environment
        self.ipython.reset()

        # Clear execution history
        self._execution_history.clear()

        # Clear VFS if auto_remove is enabled
        if self.auto_remove:
            shutil.rmtree(self.vfs.base_dir, ignore_errors=True)
            self.vfs.base_dir.mkdir(parents=True, exist_ok=True)
            self.vfs.virtual_files.clear()

        # Reset current file
        self._current_file = None

        return "Session cleared successfully"

    except Exception as e:
        return f"Clear session error: {str(e)}"

create_directory(dirpath) async

Create a new directory.

Parameters:

Name	Type	Description	Default
dirpath	str	Path of directory to create	required

Returns:

Type	Description
str	Success message

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

async def create_directory(self, dirpath: str) -> str:
    """
    Create a new directory.

    Args:
        dirpath: Path of directory to create

    Returns:
        Success message
    """
    try:
        abs_path = self.vfs.create_directory(dirpath)
        return f"Directory created successfully: {abs_path}"

    except Exception as e:
        return f"Directory creation error: {str(e)}"

execute_python(code) async

Execute Python code in the virtual environment.

Parameters:

Name	Type	Description	Default
code	str	Python code to execute	required

Returns:

Type	Description
str	Execution result as string

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

async def execute_python(self, code: str) -> str:
    """
    Execute Python code in the virtual environment.

    Args:
        code: Python code to execute

    Returns:
        Execution result as string
    """
    try:
        result = await self.ipython.run_cell(code, live_output=False)

        # Update variable manager if available
        if self.variable_manager:
            for key, value in self.ipython.user_ns.items():
                if not key.startswith('_') and key not in ['__name__', '__builtins__']:
                    try:
                        self.variable_manager.set(f"python.{key}", value)
                    except:
                        pass  # Ignore non-serializable variables

        self._execution_history.append(('python', code, result))
        return str(result) if result else "Execution completed"

    except Exception as e:
        error_msg = f"Python execution error: {str(e)}\n{traceback.format_exc()}"
        self._execution_history.append(('python', code, error_msg))
        return error_msg

execute_rust(code) async

Execute Rust code using Cargo.

Parameters:

Name	Type	Description	Default
code	str	Rust code to execute	required

Returns:

Type	Description
str	Execution result as string

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

async def execute_rust(self, code: str) -> str:
    """
    Execute Rust code using Cargo.

    Args:
        code: Rust code to execute

    Returns:
        Execution result as string
    """
    try:
        # Setup project if needed
        if not self.cargo.current_project:
            await self.cargo.setup_project("temp_rust_project")

        result = await self.cargo.run_code(code)
        self._execution_history.append(('rust', code, result))
        return result

    except Exception as e:
        error_msg = f"Rust execution error: {str(e)}"
        self._execution_history.append(('rust', code, error_msg))
        return error_msg

get_execution_history() async

Get the execution history.

Returns:

Type	Description
str	Execution history as formatted string

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

async def get_execution_history(self) -> str:
    """
    Get the execution history.

    Returns:
        Execution history as formatted string
    """
    if not self._execution_history:
        return "No execution history available."

    history_lines = []
    for i, (lang, code, result) in enumerate(self._execution_history[-10:], 1):
        history_lines.append(f"[{i}] {lang.upper()}:")
        history_lines.append(f"    Code: {code[:100]}..." if len(code) > 100 else f"    Code: {code}")
        history_lines.append(
            f"    Result: {str(result)[:200]}..." if len(str(result)) > 200 else f"    Result: {result}")
        history_lines.append("")

    return "\n".join(history_lines)

get_tools(name=None)

Get all available tools as list of tuples (function, name, description).

Returns:

Type	Description
list[tuple[Any, str, str]]	List of tool tuples

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

def get_tools(self, name:str=None) -> list[tuple[Any, str, str]]:
    """
    Get all available tools as list of tuples (function, name, description).

    Returns:
        List of tool tuples
    """
    tools = [
        # Code execution tools
        (self.execute_python, "execute_python",
         "Execute Python code in virtual environment. all variables ar available under the python scope.\n"
         "The isaa_instance is available as isaa_instance in the python code."
         " Args: code (str) -> str"),

        # (self.execute_rust, "execute_rust",
        #  "Execute Rust code using Cargo. Args: code (str) -> str"),

        # File system tools
        (self.write_file, "write_file",
         "Write content to file in virtual filesystem. lines is a string with the line range to write (e.g., '1-3' for lines 1 to 3) Args: filepath (str), content (str), lines (str) = '' -> str"),

        (self.write_file, "create_file",
         "Write content to file in virtual filesystem.  Args: filepath (str), content (str) -> str"),

        (self.replace_in_file, "replace_in_file",
         "Replace exact content in file. Args: filepath (str), old_content (str), new_content (str), precise (bool) = True -> str"),

        (self.read_file, "read_file",
         "Read content from file in virtual filesystem. lines is a string with the line range to read (e.g., '1-3' for lines 1 to 3) Args: filepath (str), lines (str) = '' -> str"),

        (self.list_files, "list_files",
         "List files in directory. Args: dirpath (str) = '.' -> str"),

        (self.list_directory, "list_directory",
         "List directory contents. Args: dirpath (str) = '.' -> str"),

        (self.create_directory, "create_directory",
         "Create new directory. Args: dirpath (str) -> str"),

        # Configuration tools
        (self.set_base_directory, "set_base_directory",
         "Set base directory for virtual filesystem. Args: path (str) -> str"),

        (self.set_current_file, "set_current_file",
         "Set current file for Python execution context. Args: filepath (str) -> str"),

        (self.install_package, "install_package",
         "Install Python package. Args: package_name (str), version (Optional[str]) -> str"),

        # Session management tools
        (self.get_execution_history, "get_execution_history",
         "Get execution history. Args: None -> str"),

        (self.clear_session, "clear_session",
         "Clear current session. Args: None -> str"),

        (self.get_variables, "get_variables",
         "Get current variables as JSON. Args: None -> str"),
    ]
    if name is not None:
        tools = [t for t in tools if t[1] == name][0]
    return tools

get_variables() async

Get current variables in JSON format.

Returns:

Type	Description
str	Variables as JSON string

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

async def get_variables(self) -> str:
    """
    Get current variables in JSON format.

    Returns:
        Variables as JSON string
    """
    try:
        # Get Python variables
        py_vars = {}
        for key, value in self.ipython.user_ns.items():
            if not key.startswith('_') and key not in ['__name__', '__builtins__']:
                try:
                    # Try to serialize the value
                    json.dumps(value, default=str)
                    py_vars[key] = str(value)[:200] if len(str(value)) > 200 else value
                except:
                    py_vars[key] = f"<{type(value).__name__}>"

        result = {
            'python_variables': py_vars,
            'current_file': self._current_file,
            'vfs_base': str(self.vfs.base_dir),
            'execution_count': len(self._execution_history)
        }

        return json.dumps(result, indent=2, default=str)

    except Exception as e:
        return f"Get variables error: {str(e)}"

install_package(package_name, version=None) async

Install a Python package in the virtual environment.

Parameters:

Name	Type	Description	Default
package_name	str	Name of the package to install	required
version	str | None	Optional specific version to install	None

Returns:

Type	Description
str	Installation result

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

    async def install_package(self, package_name: str, version: str | None = None) -> str:
        """
        Install a Python package in the virtual environment.

        Args:
            package_name: Name of the package to install
            version: Optional specific version to install

        Returns:
            Installation result
        """
        try:
            code = f"""
auto_install('{package_name}'{f", version='{version}'" if version else ""})
import {package_name.split('[')[0]}  # Import base package name
print(f"Successfully imported {package_name}")
"""
            result = await self.execute_python(code)
            return result

        except Exception as e:
            return f"Package installation error: {str(e)}"

list_directory(dirpath='.') async

List contents of a directory.

Parameters:

Name	Type	Description	Default
dirpath	str	Directory path to list	'.'

Returns:

Type	Description
str	Directory listing as string

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

async def list_directory(self, dirpath: str = '.') -> str:
    """
    List contents of a directory.

    Args:
        dirpath: Directory path to list

    Returns:
        Directory listing as string
    """
    try:
        contents = self.vfs.list_directory(dirpath)
        listing = "\n".join(f"- {item}" for item in contents)

        # Update variable manager if available
        if self.variable_manager:
            self.variable_manager.set("files.last_listing", {
                'directory': dirpath,
                'items': contents,
                'count': len(contents)
            })

        return f"Directory '{dirpath}' contents:\n{listing}"

    except Exception as e:
        return f"Directory listing error: {str(e)}"

list_files(dirpath='.') async

List files in a directory.

Parameters:

Name	Type	Description	Default
dirpath	str	Directory path to list	'.'

Returns:

Type	Description
str	File listing as string

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

async def list_files(self, dirpath: str = '.') -> str:
    """
    List files in a directory.

    Args:
        dirpath: Directory path to list

    Returns:
        File listing as string
    """
    try:
        files = self.vfs.list_files(dirpath)
        listing = "\n".join(f"- {file}" for file in files)
        return f"Files in '{dirpath}':\n{listing}"

    except Exception as e:
        return f"File listing error: {str(e)}"

read_file(filepath, lines='') async

Read content from a file in the virtual file system.

Parameters:

Name	Type	Description	Default
filepath	str	Path to the file	required
lines	str	Optional line range to read (e.g., "1-3" for lines 1 to 3)	''

Returns:

Type	Description
str	File content or error message

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

async def read_file(self, filepath: str, lines: str="") -> str:
    """
    Read content from a file in the virtual file system.

    Args:
        filepath: Path to the file
        lines: Optional line range to read (e.g., "1-3" for lines 1 to 3)

    Returns:
        File content or error message
    """
    try:
        content = self.vfs.read_file(filepath)

        if lines:
            start, end = map(int, lines.split('-'))
            content = '\n'.join(content.split('\n')[start-1:end])
        # Update variable manager if available
        if self.variable_manager:
            self.variable_manager.set("files.last_read", {
                'path': filepath,
                'size': len(content),
                'content_preview': content[:200] + '...' if len(content) > 200 else content
            })

        return content

    except Exception as e:
        return f"File read error: {str(e)}"

replace_in_file(filepath, old_content, new_content, precise=True) async

Replace exact content in file with new content.

Parameters:

Name	Type	Description	Default
filepath	str	Path to the file	required
old_content	str	Exact content to replace (empty string for insertion at start)	required
new_content	str	Content to replace with	required
precise	bool	If True, requires exact match; if False, allows single occurrence replacement	True

Returns:

Type	Description
str	Success message or error

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

async def replace_in_file(self, filepath: str, old_content: str, new_content: str, precise: bool = True) -> str:
    """
    Replace exact content in file with new content.

    Args:
        filepath: Path to the file
        old_content: Exact content to replace (empty string for insertion at start)
        new_content: Content to replace with
        precise: If True, requires exact match; if False, allows single occurrence replacement

    Returns:
        Success message or error
    """
    try:
        # Read current file content
        try:
            current_content = self.vfs.read_file(filepath)
        except:
            return f"Error: File '{filepath}' not found or cannot be read"

        # Handle insertion at start (empty old_content)
        if not old_content:
            updated_content = new_content + current_content
            self.vfs.write_file(filepath, updated_content)
            return f"Content inserted at start of '{filepath}'"

        # Check if old_content exists
        if old_content not in current_content:
            return f"Error: Old content not found in '{filepath}' use read_file to check."

        # Count occurrences
        occurrences = current_content.count(old_content)

        if precise and occurrences > 1:
            return f"Error: Found {occurrences} occurrences of old content. Use precise=False to replace first occurrence."

        # Replace content (first occurrence if multiple)
        updated_content = current_content.replace(old_content, new_content, 1)

        # Write updated content
        self.vfs.write_file(filepath, updated_content)

        return f"Successfully replaced content in '{filepath}' ({occurrences} occurrence{'s' if occurrences > 1 else ''} found, 1 replaced)"

    except Exception as e:
        return f"Replace error: {str(e)}"

set_base_directory(path)

Set the base directory for the virtual file system.

Parameters:

Name	Type	Description	Default
path	str	New base directory path	required

Returns:

Type	Description
str	Success message

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

def set_base_directory(self, path: str) -> str:
    """
    Set the base directory for the virtual file system.

    Args:
        path: New base directory path

    Returns:
        Success message
    """
    try:
        new_path = Path(path) if isinstance(path, str) else path
        new_path = new_path.absolute()
        print(f"New path: {new_path}")
        new_path.mkdir(parents=True, exist_ok=True)
        self.vfs.base_dir = new_path
        self.vfs.current_dir = new_path

        # Update MockIPython base directory and sys.path
        result = self.ipython.set_base_directory(path)

        return result

    except Exception as e:
        return f"Set base directory error: {str(e)}"

set_current_file(filepath) async

Set the current file for Python execution context.

Parameters:

Name	Type	Description	Default
filepath	str	Path to set as current file	required

Returns:

Type	Description
str	Success message

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

async def set_current_file(self, filepath: str) -> str:
    """
    Set the current file for Python execution context.

    Args:
        filepath: Path to set as current file

    Returns:
        Success message
    """
    try:
        abs_path = self.vfs._resolve_path(filepath)
        self.ipython.user_ns['__file__'] = str(abs_path)
        self._current_file = str(abs_path)

        return f"Current file set to: {abs_path}"

    except Exception as e:
        return f"Set current file error: {str(e)}"

write_file(filepath, content, lines='') async

Write content to a file in the virtual file system.

Parameters:

Name	Type	Description	Default
filepath	str	Path to the file	required
content	str	Content to write	required
lines	str	Optional line range to write (e.g., "1-3" for lines 1 to 3)	''

Returns:

Type	Description
str	Success message

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

async def write_file(self, filepath: str, content: str, lines: str = "") -> str:
    """
    Write content to a file in the virtual file system.

    Args:
        filepath: Path to the file
        content: Content to write
        lines: Optional line range to write (e.g., "1-3" for lines 1 to 3)

    Returns:
        Success message
    """

    try:
        if lines:
            abs_path = self.vfs.overwrite_lines(filepath, content, lines)
        else:
            abs_path = self.vfs.write_file(filepath, content)

        # Update variable manager if available
        if self.variable_manager:
            self.variable_manager.set(f"files.{filepath.replace('/', '.')}", {
                'path': str(abs_path),
                'size': len(content),
                'content_preview': content[:100] + '...' if len(content) > 100 else content
            })

        return f"File written successfully: {abs_path}"

    except Exception as e:
        return f"File write error: {str(e)}"

VirtualEnvContext

Context manager for temporary virtual environment activation

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

class VirtualEnvContext:
    """Context manager for temporary virtual environment activation"""

    def __init__(self, venv_path: Path):
        self.venv_path = venv_path
        self._original_path = None
        self._original_sys_path = None
        self._original_prefix = None
        self._original_virtual_env = None

    def _get_venv_paths(self):
        """Get virtual environment paths based on platform"""
        if sys.platform == 'win32':
            site_packages = self.venv_path / 'Lib' / 'site-packages'
            scripts_dir = self.venv_path / 'Scripts'
            python_path = scripts_dir / 'python.exe'
        else:
            python_version = f'python{sys.version_info.major}.{sys.version_info.minor}'
            site_packages = self.venv_path / 'lib' / python_version / 'site-packages'
            scripts_dir = self.venv_path / 'bin'
            python_path = scripts_dir / 'python'

        return site_packages, scripts_dir, python_path

    def __enter__(self):
        # Save original state
        self._original_path = os.environ.get('PATH', '')
        self._original_sys_path = sys.path.copy()
        self._original_prefix = sys.prefix
        self._original_virtual_env = os.environ.get('VIRTUAL_ENV')

        # Get venv paths
        site_packages, scripts_dir, python_path = self._get_venv_paths()

        # Modify environment for venv
        if scripts_dir.exists():
            new_path = os.pathsep.join([str(scripts_dir), self._original_path])
            os.environ['PATH'] = new_path

        if site_packages.exists():
            sys.path.insert(0, str(site_packages))

        os.environ['VIRTUAL_ENV'] = str(self.venv_path)

        # Return the python executable path for potential subprocess calls
        return str(python_path)

    def __exit__(self, exc_type, exc_val, exc_tb):
        # Restore original state
        os.environ['PATH'] = self._original_path
        sys.path = self._original_sys_path

        if self._original_virtual_env is None:
            os.environ.pop('VIRTUAL_ENV', None)
        else:
            os.environ['VIRTUAL_ENV'] = self._original_virtual_env

VirtualFileSystem

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

class VirtualFileSystem:
    def __init__(self, base_dir: Path):
        self.base_dir = base_dir
        self.current_dir = base_dir
        self.virtual_files: dict[str, str] = {}
        self.base_dir.mkdir(parents=True, exist_ok=True)

    def ifile_exists(self, filepath: str | Path) -> bool:
        """Check if a file exists"""
        abs_path = self._resolve_path(filepath)
        return abs_path.exists()

    def overwrite_lines(self, filepath: str | Path, new_content: str, lines: str = "") -> str:
        """Overwrite lines in a file"""
        try:
            content = self.read_file(filepath)
            content_lines = content.split('\n')
            start, end = map(int, lines.split('-'))
            # overwrite specif lines with new content keep the rest
            content_before = content_lines[:start-1]
            content_after = content_lines[end:]
            content_lines = content_before + new_content.split('\n') + content_after
            content = '\n'.join(content_lines)
            return self.write_file(filepath, content)
        except Exception as e:
            return f"Overwrite lines failed: {str(e)}"

    def write_file(self, filepath: str | Path, content: str) -> Path:
        """Write content to a virtual file and persist to disk using UTF-8"""
        try:
            abs_path = self._resolve_path(filepath)
        except ValueError:
            print("invalid :", filepath)
            filepath = "src/temp/_temp_fix.py"
            abs_path = self._resolve_path(filepath)
        abs_path.parent.mkdir(parents=True, exist_ok=True)

        # Store in virtual filesystem
        rel_path = str(abs_path.relative_to(self.base_dir))
        self.virtual_files[rel_path] = content

        # Write to actual filesystem with UTF-8 encoding
        with open(abs_path, 'w', encoding='utf-8', errors='replace') as f:
            f.write(content)

        parent_dir_str = str(abs_path.parent.absolute())
        if parent_dir_str not in sys.path and abs_path.suffix == '.py':
            sys.path.insert(0, parent_dir_str)

        return abs_path

    def read_file(self, filepath: str | Path) -> str:
        """Read content from a virtual file using UTF-8"""
        abs_path = self._resolve_path(filepath)
        if not abs_path.exists():
            raise FileNotFoundError(f"File not found: {filepath}")

        rel_path = str(abs_path.relative_to(self.base_dir))

        # Check virtual filesystem first
        if rel_path in self.virtual_files:
            return self.virtual_files[rel_path]

        # Fall back to reading from disk with UTF-8 encoding
        with open(abs_path, encoding='utf-8', errors='replace') as f:
            content = f.read()
            self.virtual_files[rel_path] = content
            return content

    def delete_file(self, filepath: str | Path):
        """Delete a virtual file"""
        abs_path = self._resolve_path(filepath)
        rel_path = str(abs_path.relative_to(self.base_dir))

        if rel_path in self.virtual_files:
            del self.virtual_files[rel_path]

        if abs_path.exists():
            abs_path.unlink()

    def create_directory(self, dirpath: str | Path):
        """Create a new directory"""
        abs_path = self._resolve_path(dirpath)
        abs_path.mkdir(parents=True, exist_ok=True)
        return abs_path

    def list_files(self, dirpath: str | Path = '.') -> list:
        """List files in a directory"""
        abs_path = self._resolve_path(dirpath)
        if not abs_path.exists():
            raise FileNotFoundError(f"Directory not found: {dirpath}")
        return [p.name for p in abs_path.iterdir() if p.is_file()]

    def list_directory(self, dirpath: str | Path = '.') -> list:
        """List contents of a directory"""
        abs_path = self._resolve_path(dirpath)
        if not abs_path.exists():
            raise FileNotFoundError(f"Directory not found: {dirpath}")
        return [p.name for p in abs_path.iterdir()]

    def change_directory(self, dirpath: str | Path):
        """Change current working directory"""
        new_dir = self._resolve_path(dirpath)
        if not new_dir.exists() or not new_dir.is_dir():
            raise NotADirectoryError(f"Directory not found: {dirpath}")
        self.current_dir = new_dir

    def _resolve_path(self, filepath: str | Path) -> Path:
        """Convert relative path to absolute path"""
        filepath = Path(filepath)
        if filepath.is_absolute():
            if not str(filepath).startswith(str(self.base_dir)):
                raise ValueError("Path must be within base directory")
            return filepath
        return (self.current_dir / filepath).resolve()

    def save_state(self, state_file: Path):
        """Save virtual filesystem state to disk"""
        state = {
            'current_dir': str(self.current_dir.relative_to(self.base_dir)),
            'virtual_files': self.virtual_files
        }
        with open(state_file, 'w') as f:
            json.dump(state, f)

    def load_state(self, state_file: Path):
        """Load virtual filesystem state from disk"""
        if not state_file.exists():
            return

        with open(state_file) as f:
            state = json.load(f)
            self.current_dir = self.base_dir / state['current_dir']
            self.virtual_files = state['virtual_files']

    def print_file_structure(self, start_path: str | Path = '.', indent: str = ''):
        """Print the file structure starting from the given path"""
        start_path = self._resolve_path(start_path)
        if not start_path.exists():
            s = f"Path not found: {start_path}"
            return s

        s = f"{indent}{start_path.name}/"
        for item in sorted(start_path.iterdir()):
            if item.is_dir():
               s+= self.print_file_structure(item, indent + '  ')
            else:
                s = f"{indent}  {item.name}"
        return s

change_directory(dirpath)

Change current working directory

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

def change_directory(self, dirpath: str | Path):
    """Change current working directory"""
    new_dir = self._resolve_path(dirpath)
    if not new_dir.exists() or not new_dir.is_dir():
        raise NotADirectoryError(f"Directory not found: {dirpath}")
    self.current_dir = new_dir

create_directory(dirpath)

Create a new directory

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

def create_directory(self, dirpath: str | Path):
    """Create a new directory"""
    abs_path = self._resolve_path(dirpath)
    abs_path.mkdir(parents=True, exist_ok=True)
    return abs_path

delete_file(filepath)

Delete a virtual file

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

def delete_file(self, filepath: str | Path):
    """Delete a virtual file"""
    abs_path = self._resolve_path(filepath)
    rel_path = str(abs_path.relative_to(self.base_dir))

    if rel_path in self.virtual_files:
        del self.virtual_files[rel_path]

    if abs_path.exists():
        abs_path.unlink()

ifile_exists(filepath)

Check if a file exists

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

def ifile_exists(self, filepath: str | Path) -> bool:
    """Check if a file exists"""
    abs_path = self._resolve_path(filepath)
    return abs_path.exists()

list_directory(dirpath='.')

List contents of a directory

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

def list_directory(self, dirpath: str | Path = '.') -> list:
    """List contents of a directory"""
    abs_path = self._resolve_path(dirpath)
    if not abs_path.exists():
        raise FileNotFoundError(f"Directory not found: {dirpath}")
    return [p.name for p in abs_path.iterdir()]

list_files(dirpath='.')

List files in a directory

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

def list_files(self, dirpath: str | Path = '.') -> list:
    """List files in a directory"""
    abs_path = self._resolve_path(dirpath)
    if not abs_path.exists():
        raise FileNotFoundError(f"Directory not found: {dirpath}")
    return [p.name for p in abs_path.iterdir() if p.is_file()]

load_state(state_file)

Load virtual filesystem state from disk

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

def load_state(self, state_file: Path):
    """Load virtual filesystem state from disk"""
    if not state_file.exists():
        return

    with open(state_file) as f:
        state = json.load(f)
        self.current_dir = self.base_dir / state['current_dir']
        self.virtual_files = state['virtual_files']

overwrite_lines(filepath, new_content, lines='')

Overwrite lines in a file

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

def overwrite_lines(self, filepath: str | Path, new_content: str, lines: str = "") -> str:
    """Overwrite lines in a file"""
    try:
        content = self.read_file(filepath)
        content_lines = content.split('\n')
        start, end = map(int, lines.split('-'))
        # overwrite specif lines with new content keep the rest
        content_before = content_lines[:start-1]
        content_after = content_lines[end:]
        content_lines = content_before + new_content.split('\n') + content_after
        content = '\n'.join(content_lines)
        return self.write_file(filepath, content)
    except Exception as e:
        return f"Overwrite lines failed: {str(e)}"

print_file_structure(start_path='.', indent='')

Print the file structure starting from the given path

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

def print_file_structure(self, start_path: str | Path = '.', indent: str = ''):
    """Print the file structure starting from the given path"""
    start_path = self._resolve_path(start_path)
    if not start_path.exists():
        s = f"Path not found: {start_path}"
        return s

    s = f"{indent}{start_path.name}/"
    for item in sorted(start_path.iterdir()):
        if item.is_dir():
           s+= self.print_file_structure(item, indent + '  ')
        else:
            s = f"{indent}  {item.name}"
    return s

read_file(filepath)

Read content from a virtual file using UTF-8

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

def read_file(self, filepath: str | Path) -> str:
    """Read content from a virtual file using UTF-8"""
    abs_path = self._resolve_path(filepath)
    if not abs_path.exists():
        raise FileNotFoundError(f"File not found: {filepath}")

    rel_path = str(abs_path.relative_to(self.base_dir))

    # Check virtual filesystem first
    if rel_path in self.virtual_files:
        return self.virtual_files[rel_path]

    # Fall back to reading from disk with UTF-8 encoding
    with open(abs_path, encoding='utf-8', errors='replace') as f:
        content = f.read()
        self.virtual_files[rel_path] = content
        return content

save_state(state_file)

Save virtual filesystem state to disk

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

def save_state(self, state_file: Path):
    """Save virtual filesystem state to disk"""
    state = {
        'current_dir': str(self.current_dir.relative_to(self.base_dir)),
        'virtual_files': self.virtual_files
    }
    with open(state_file, 'w') as f:
        json.dump(state, f)

write_file(filepath, content)

Write content to a virtual file and persist to disk using UTF-8

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

def write_file(self, filepath: str | Path, content: str) -> Path:
    """Write content to a virtual file and persist to disk using UTF-8"""
    try:
        abs_path = self._resolve_path(filepath)
    except ValueError:
        print("invalid :", filepath)
        filepath = "src/temp/_temp_fix.py"
        abs_path = self._resolve_path(filepath)
    abs_path.parent.mkdir(parents=True, exist_ok=True)

    # Store in virtual filesystem
    rel_path = str(abs_path.relative_to(self.base_dir))
    self.virtual_files[rel_path] = content

    # Write to actual filesystem with UTF-8 encoding
    with open(abs_path, 'w', encoding='utf-8', errors='replace') as f:
        f.write(content)

    parent_dir_str = str(abs_path.parent.absolute())
    if parent_dir_str not in sys.path and abs_path.suffix == '.py':
        sys.path.insert(0, parent_dir_str)

    return abs_path

auto_install(package_name, install_method='pip', upgrade=False, quiet=False, version=None, extra_args=None)

Enhanced auto-save import with version and extra arguments support

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

def auto_install(package_name, install_method='pip', upgrade=False, quiet=False, version=None, extra_args=None):
    '''
    Enhanced auto-save import with version and extra arguments support
    '''
    try:
        # Attempt to import the package
        return importlib.import_module(package_name)
    except ImportError:
        # Package not found, prepare for installation
        print(f"Package '{package_name}' not found. Attempting to install...")
        try:
            # Determine Python executable based on virtual environment
            venv_path = os.environ.get('VIRTUAL_ENV')
            if venv_path:
                venv_path = Path(venv_path)
                if sys.platform == 'win32':
                    python_exec = str(venv_path / 'Scripts' / 'python.exe')
                else:
                    python_exec = str(venv_path / 'bin' / 'python')
                # Check if the Python executable exists
                if not Path(python_exec).exists():
                    python_exec = sys.executable
            else:
                python_exec = sys.executable

            # Construct installation command with more flexibility
            install_cmd = [python_exec, "-m", install_method, "install"]
            if upgrade:
                install_cmd.append("--upgrade")
            # Support specific version installation
            if version:
                install_cmd.append(f"{package_name}=={version}")
            else:
                install_cmd.append(package_name)
            # Add extra arguments if provided
            if extra_args:
                install_cmd.extend(extra_args)
            # Run installation with appropriate verbosity
            installation_output = subprocess.run(
                install_cmd,
                capture_output=quiet,
                text=True
            )
            # Check installation status
            if installation_output.returncode == 0:
                print(f"Successfully installed {package_name}")
                return importlib.import_module(package_name)
            else:
                raise Exception(f"Installation failed: {installation_output.stderr}")
        except Exception as install_error:
            print(f"Error installing {package_name}: {install_error}")
            return None

sync_globals_to_vars(pipeline, namespace=None, prefix=None, include_types=None, exclude_patterns=None, exclude_private=True, deep_copy=False, only_serializable=False)

Sync global variables or a specific namespace to pipeline variables.

Args:
    pipeline: Pipeline instance to sync variables to
    namespace: Optional dictionary of variables (defaults to globals())
    prefix: Optional prefix for variable names (e.g., 'global_')
    include_types: Only include variables of these types
    exclude_patterns: List of regex patterns to exclude
    exclude_private: Exclude variables starting with underscore
    deep_copy: Create deep copies of variables instead of references
    only_serializable: Only include variables that can be serialized

Returns:
    SyncReport with details about added, skipped and error variables

Usage example:

Basic usage - sync all globals

report = sync_globals_to_vars(pipeline)

Sync only numeric types with prefix

report = sync_globals_to_vars( pipeline, include_types=[int, float], prefix="global_" )

Sync from specific namespace

import numpy as np namespace = {"arr": np.array([1,2,3])} report = sync_globals_to_vars(pipeline, namespace=namespace)

Sync with deep copy and serialization check

report = sync_globals_to_vars( pipeline, deep_copy=True, only_serializable=True )

Source code in toolboxv2/mods/isaa/CodingAgent/live.py

def sync_globals_to_vars(
    pipeline: Any,
    namespace: dict[str, Any] | None = None,
    prefix: str | None = None,
    include_types: type | list[type] | None = None,
    exclude_patterns: list[str] | None = None,
    exclude_private: bool = True,
    deep_copy: bool = False,
    only_serializable: bool = False
) -> SyncReport:
    """
    Sync global variables or a specific namespace to pipeline variables.

    Args:
        pipeline: Pipeline instance to sync variables to
        namespace: Optional dictionary of variables (defaults to globals())
        prefix: Optional prefix for variable names (e.g., 'global_')
        include_types: Only include variables of these types
        exclude_patterns: List of regex patterns to exclude
        exclude_private: Exclude variables starting with underscore
        deep_copy: Create deep copies of variables instead of references
        only_serializable: Only include variables that can be serialized

    Returns:
        SyncReport with details about added, skipped and error variables

    Usage example:
# Basic usage - sync all globals
report = sync_globals_to_vars(pipeline)

# Sync only numeric types with prefix
report = sync_globals_to_vars(
    pipeline,
    include_types=[int, float],
    prefix="global_"
)

# Sync from specific namespace
import numpy as np
namespace = {"arr": np.array([1,2,3])}
report = sync_globals_to_vars(pipeline, namespace=namespace)

# Sync with deep copy and serialization check
report = sync_globals_to_vars(
    pipeline,
    deep_copy=True,
    only_serializable=True
)
    """
    # Initialize report
    report = SyncReport(
        added={},
        skipped={},
        errors={}
    )

    # Get namespace
    if namespace is None:
        # Get caller's globals
        namespace = currentframe().f_back.f_globals

    # Compile exclude patterns
    if exclude_patterns:
        patterns = [re.compile(pattern) for pattern in exclude_patterns]
    else:
        patterns = []

    # Normalize include_types
    if include_types and not isinstance(include_types, list | tuple | set):
        include_types = [include_types]
    def get_type_info(var: Any) -> str:
        """Helper to get detailed type information"""
        if isinstance(var, type):
            return f"class '{var.__name__}'"
        elif isinstance(var, BaseModel):
            return f"Pydantic model '{var.__class__.__name__}'"
        elif hasattr(var, '__class__'):
            type_name = var.__class__.__name__
            module_name = var.__class__.__module__
            if module_name != 'builtins':
                return f"{module_name}.{type_name}"
            return type_name
        return type(var).__name__
    # Process each variable
    for name, value in namespace.items():
        try:
            # Skip if matches exclude criteria
            if exclude_private and name.startswith('_'):
                report.skipped[name] = "private variable"
                continue

            if any(pattern.match(name) for pattern in patterns):
                report.skipped[name] = "matched exclude pattern"
                continue

            if include_types and not isinstance(value, tuple(include_types)):
                report.skipped[name] = f"type {type(value).__name__} not in include_types"
                continue

            # Test serialization if required
            if only_serializable:
                try:
                    import pickle
                    pickle.dumps(value)
                except Exception as e:
                    report.skipped[name] = f"not serializable: {str(e)}"
                    continue

            # Prepare variable
            var_value = deepcopy(value) if deep_copy else value
            var_name = f"{prefix}{name}" if prefix else name

            # Add to pipeline variables
            pipeline.variables[var_name] = var_value
            report.added[var_name] = get_type_info(value)

        except Exception as e:
            report.errors[name] = str(e)

    return report

base

Agent

agent

AgentCheckpoint dataclass

Enhanced AgentCheckpoint with UnifiedContextManager and ChatSession integration

Source code in toolboxv2/mods/isaa/base/Agent/types.py

@dataclass
class AgentCheckpoint:
    """Enhanced AgentCheckpoint with UnifiedContextManager and ChatSession integration"""
    timestamp: datetime
    agent_state: dict[str, Any]
    task_state: dict[str, Any]
    world_model: dict[str, Any]
    active_flows: list[str]
    metadata: dict[str, Any] = field(default_factory=dict)

    # NEUE: Enhanced checkpoint data for UnifiedContextManager integration
    session_data: dict[str, Any] = field(default_factory=dict)
    context_manager_state: dict[str, Any] = field(default_factory=dict)
    conversation_history: list[dict[str, Any]] = field(default_factory=list)
    variable_system_state: dict[str, Any] = field(default_factory=dict)
    results_store: dict[str, Any] = field(default_factory=dict)
    tool_capabilities: dict[str, Any] = field(default_factory=dict)
    variable_scopes: dict[str, Any] = field(default_factory=dict)

    # Optional: Additional system state
    performance_metrics: dict[str, Any] = field(default_factory=dict)
    execution_history: list[dict[str, Any]] = field(default_factory=list)

    def get_checkpoint_summary(self) -> str:
        """Get human-readable checkpoint summary"""
        try:
            summary_parts = []

            # Basic info
            if self.session_data:
                session_count = len([s for s in self.session_data.values() if s.get("status") != "failed"])
                summary_parts.append(f"{session_count} sessions")

            # Task info
            if self.task_state:
                completed_tasks = len([t for t in self.task_state.values() if t.get("status") == "completed"])
                total_tasks = len(self.task_state)
                summary_parts.append(f"{completed_tasks}/{total_tasks} tasks")

            # Conversation info
            if self.conversation_history:
                summary_parts.append(f"{len(self.conversation_history)} messages")

            # Context info
            if self.context_manager_state:
                cache_count = self.context_manager_state.get("cache_entries", 0)
                if cache_count > 0:
                    summary_parts.append(f"{cache_count} cached contexts")

            # Variable system info
            if self.variable_system_state:
                scopes = len(self.variable_system_state.get("scopes", {}))
                summary_parts.append(f"{scopes} variable scopes")

            # Tool capabilities
            if self.tool_capabilities:
                summary_parts.append(f"{len(self.tool_capabilities)} analyzed tools")

            return "; ".join(summary_parts) if summary_parts else "Basic checkpoint"

        except Exception as e:
            return f"Summary generation failed: {str(e)}"

    def get_storage_size_estimate(self) -> dict[str, int]:
        """Estimate storage size of different checkpoint components"""
        try:
            sizes = {}

            # Calculate sizes in bytes (approximate)
            sizes["agent_state"] = len(str(self.agent_state))
            sizes["task_state"] = len(str(self.task_state))
            sizes["world_model"] = len(str(self.world_model))
            sizes["conversation_history"] = len(str(self.conversation_history))
            sizes["session_data"] = len(str(self.session_data))
            sizes["context_manager_state"] = len(str(self.context_manager_state))
            sizes["variable_system_state"] = len(str(self.variable_system_state))
            sizes["results_store"] = len(str(self.results_store))
            sizes["tool_capabilities"] = len(str(self.tool_capabilities))

            sizes["total_bytes"] = sum(sizes.values())
            sizes["total_kb"] = sizes["total_bytes"] / 1024
            sizes["total_mb"] = sizes["total_kb"] / 1024

            return sizes

        except Exception as e:
            return {"error": str(e)}

    def validate_checkpoint_integrity(self) -> dict[str, Any]:
        """Validate checkpoint integrity and completeness"""
        validation = {
            "is_valid": True,
            "errors": [],
            "warnings": [],
            "completeness_score": 0.0,
            "components_present": []
        }

        try:
            # Check required components
            required_components = ["timestamp", "agent_state", "task_state", "world_model", "active_flows"]
            for component in required_components:
                if hasattr(self, component) and getattr(self, component) is not None:
                    validation["components_present"].append(component)
                else:
                    validation["errors"].append(f"Missing required component: {component}")
                    validation["is_valid"] = False

            # Check optional enhanced components
            enhanced_components = ["session_data", "context_manager_state", "conversation_history",
                                   "variable_system_state", "results_store", "tool_capabilities"]

            for component in enhanced_components:
                if hasattr(self, component) and getattr(self, component):
                    validation["components_present"].append(component)

            # Calculate completeness score
            total_possible = len(required_components) + len(enhanced_components)
            validation["completeness_score"] = len(validation["components_present"]) / total_possible

            # Check timestamp validity
            if isinstance(self.timestamp, datetime):
                age_hours = (datetime.now() - self.timestamp).total_seconds() / 3600
                if age_hours > 24:
                    validation["warnings"].append(f"Checkpoint is {age_hours:.1f} hours old")
            else:
                validation["errors"].append("Invalid timestamp format")
                validation["is_valid"] = False

            # Check session data consistency
            if self.session_data and self.conversation_history:
                session_ids_in_data = set(self.session_data.keys())
                session_ids_in_conversation = set(
                    msg.get("session_id") for msg in self.conversation_history
                    if msg.get("session_id")
                )

                if session_ids_in_data != session_ids_in_conversation:
                    validation["warnings"].append("Session data and conversation history session IDs don't match")

            return validation

        except Exception as e:
            validation["errors"].append(f"Validation error: {str(e)}")
            validation["is_valid"] = False
            return validation

    def get_version_info(self) -> dict[str, str]:
        """Get checkpoint version information"""
        return {
            "checkpoint_version": self.metadata.get("checkpoint_version", "1.0"),
            "data_format": "enhanced" if self.session_data or self.context_manager_state else "basic",
            "context_system": "unified" if self.context_manager_state else "legacy",
            "variable_system": "integrated" if self.variable_system_state else "basic",
            "session_management": "chatsession" if self.session_data else "memory_only",
            "created_with": "FlowAgent v2.0 Enhanced Context System"
        }

get_checkpoint_summary()

Get human-readable checkpoint summary

Source code in toolboxv2/mods/isaa/base/Agent/types.py

def get_checkpoint_summary(self) -> str:
    """Get human-readable checkpoint summary"""
    try:
        summary_parts = []

        # Basic info
        if self.session_data:
            session_count = len([s for s in self.session_data.values() if s.get("status") != "failed"])
            summary_parts.append(f"{session_count} sessions")

        # Task info
        if self.task_state:
            completed_tasks = len([t for t in self.task_state.values() if t.get("status") == "completed"])
            total_tasks = len(self.task_state)
            summary_parts.append(f"{completed_tasks}/{total_tasks} tasks")

        # Conversation info
        if self.conversation_history:
            summary_parts.append(f"{len(self.conversation_history)} messages")

        # Context info
        if self.context_manager_state:
            cache_count = self.context_manager_state.get("cache_entries", 0)
            if cache_count > 0:
                summary_parts.append(f"{cache_count} cached contexts")

        # Variable system info
        if self.variable_system_state:
            scopes = len(self.variable_system_state.get("scopes", {}))
            summary_parts.append(f"{scopes} variable scopes")

        # Tool capabilities
        if self.tool_capabilities:
            summary_parts.append(f"{len(self.tool_capabilities)} analyzed tools")

        return "; ".join(summary_parts) if summary_parts else "Basic checkpoint"

    except Exception as e:
        return f"Summary generation failed: {str(e)}"

get_storage_size_estimate()

Estimate storage size of different checkpoint components

Source code in toolboxv2/mods/isaa/base/Agent/types.py

def get_storage_size_estimate(self) -> dict[str, int]:
    """Estimate storage size of different checkpoint components"""
    try:
        sizes = {}

        # Calculate sizes in bytes (approximate)
        sizes["agent_state"] = len(str(self.agent_state))
        sizes["task_state"] = len(str(self.task_state))
        sizes["world_model"] = len(str(self.world_model))
        sizes["conversation_history"] = len(str(self.conversation_history))
        sizes["session_data"] = len(str(self.session_data))
        sizes["context_manager_state"] = len(str(self.context_manager_state))
        sizes["variable_system_state"] = len(str(self.variable_system_state))
        sizes["results_store"] = len(str(self.results_store))
        sizes["tool_capabilities"] = len(str(self.tool_capabilities))

        sizes["total_bytes"] = sum(sizes.values())
        sizes["total_kb"] = sizes["total_bytes"] / 1024
        sizes["total_mb"] = sizes["total_kb"] / 1024

        return sizes

    except Exception as e:
        return {"error": str(e)}

get_version_info()

Get checkpoint version information

Source code in toolboxv2/mods/isaa/base/Agent/types.py

def get_version_info(self) -> dict[str, str]:
    """Get checkpoint version information"""
    return {
        "checkpoint_version": self.metadata.get("checkpoint_version", "1.0"),
        "data_format": "enhanced" if self.session_data or self.context_manager_state else "basic",
        "context_system": "unified" if self.context_manager_state else "legacy",
        "variable_system": "integrated" if self.variable_system_state else "basic",
        "session_management": "chatsession" if self.session_data else "memory_only",
        "created_with": "FlowAgent v2.0 Enhanced Context System"
    }

validate_checkpoint_integrity()

Validate checkpoint integrity and completeness

Source code in toolboxv2/mods/isaa/base/Agent/types.py

def validate_checkpoint_integrity(self) -> dict[str, Any]:
    """Validate checkpoint integrity and completeness"""
    validation = {
        "is_valid": True,
        "errors": [],
        "warnings": [],
        "completeness_score": 0.0,
        "components_present": []
    }

    try:
        # Check required components
        required_components = ["timestamp", "agent_state", "task_state", "world_model", "active_flows"]
        for component in required_components:
            if hasattr(self, component) and getattr(self, component) is not None:
                validation["components_present"].append(component)
            else:
                validation["errors"].append(f"Missing required component: {component}")
                validation["is_valid"] = False

        # Check optional enhanced components
        enhanced_components = ["session_data", "context_manager_state", "conversation_history",
                               "variable_system_state", "results_store", "tool_capabilities"]

        for component in enhanced_components:
            if hasattr(self, component) and getattr(self, component):
                validation["components_present"].append(component)

        # Calculate completeness score
        total_possible = len(required_components) + len(enhanced_components)
        validation["completeness_score"] = len(validation["components_present"]) / total_possible

        # Check timestamp validity
        if isinstance(self.timestamp, datetime):
            age_hours = (datetime.now() - self.timestamp).total_seconds() / 3600
            if age_hours > 24:
                validation["warnings"].append(f"Checkpoint is {age_hours:.1f} hours old")
        else:
            validation["errors"].append("Invalid timestamp format")
            validation["is_valid"] = False

        # Check session data consistency
        if self.session_data and self.conversation_history:
            session_ids_in_data = set(self.session_data.keys())
            session_ids_in_conversation = set(
                msg.get("session_id") for msg in self.conversation_history
                if msg.get("session_id")
            )

            if session_ids_in_data != session_ids_in_conversation:
                validation["warnings"].append("Session data and conversation history session IDs don't match")

        return validation

    except Exception as e:
        validation["errors"].append(f"Validation error: {str(e)}")
        validation["is_valid"] = False
        return validation

AgentModelData

Bases: BaseModel

Source code in toolboxv2/mods/isaa/base/Agent/types.py

class AgentModelData(BaseModel):
    name: str = "FlowAgent"
    fast_llm_model: str = "openrouter/anthropic/claude-3-haiku"
    complex_llm_model: str = "openrouter/openai/gpt-4o"
    system_message: str = "You are a production-ready autonomous agent."
    temperature: float = 0.7
    max_tokens: int = 2048
    max_input_tokens: int = 32768
    api_key: str | None  = None
    api_base: str | None  = None
    budget_manager: Any  = None
    caching: bool = True
    persona: PersonaConfig | None = True
    use_fast_response: bool = True

    def get_system_message_with_persona(self) -> str:
        """Get system message with persona integration"""
        base_message = self.system_message

        if self.persona and self.persona.apply_method in ["system_prompt", "both"]:
            persona_addition = self.persona.to_system_prompt_addition()
            if persona_addition:
                base_message += f"\n## Persona Instructions\n{persona_addition}"

        return base_message

get_system_message_with_persona()

Get system message with persona integration

Source code in toolboxv2/mods/isaa/base/Agent/types.py

def get_system_message_with_persona(self) -> str:
    """Get system message with persona integration"""
    base_message = self.system_message

    if self.persona and self.persona.apply_method in ["system_prompt", "both"]:
        persona_addition = self.persona.to_system_prompt_addition()
        if persona_addition:
            base_message += f"\n## Persona Instructions\n{persona_addition}"

    return base_message

BindingSyncHandler

Handles automatic synchronization between bound agents

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

class BindingSyncHandler:
    """Handles automatic synchronization between bound agents"""

    def __init__(self, binding_config: dict):
        self.binding_config = binding_config
        self.sync_queue = []
        self.last_sync = time.time()

    def cleanup(self):
        """Clean up sync handler resources"""
        self.sync_queue.clear()
        rprint(f"Binding sync handler for {self.binding_config['binding_id']} cleaned up")

cleanup()

Clean up sync handler resources

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

def cleanup(self):
    """Clean up sync handler resources"""
    self.sync_queue.clear()
    rprint(f"Binding sync handler for {self.binding_config['binding_id']} cleaned up")

ChainMetadata dataclass

Metadata for stored chains

Source code in toolboxv2/mods/isaa/base/Agent/types.py

@dataclass
class ChainMetadata:
    """Metadata for stored chains"""
    name: str
    description: str = ""
    created_at: datetime = field(default_factory=datetime.now)
    modified_at: datetime = field(default_factory=datetime.now)
    version: str = "1.0.0"
    tags: list[str] = field(default_factory=list)
    author: str = ""
    complexity: str = "simple"  # simple, medium, complex
    agent_count: int = 0
    has_conditionals: bool = False
    has_parallels: bool = False
    has_error_handling: bool = False

CompletionCheckerNode

Bases: AsyncNode

Breaks infinite cycles by checking actual completion status

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

@with_progress_tracking
class CompletionCheckerNode(AsyncNode):
    """Breaks infinite cycles by checking actual completion status"""

    def __init__(self):
        super().__init__()
        self.execution_count = 0
        self.max_cycles = 5  # Prevent infinite loops

    async def prep_async(self, shared):
        current_plan = shared.get("current_plan")
        tasks = shared.get("tasks", {})

        return {
            "current_plan": current_plan,
            "tasks": tasks,
            "execution_count": self.execution_count
        }

    async def exec_async(self, prep_res):
        self.execution_count += 1

        # Safety check: prevent infinite loops
        if self.execution_count > self.max_cycles:
            wprint(f"Max execution cycles ({self.max_cycles}) reached, terminating")
            return {
                "action": "force_terminate",
                "reason": "Max cycles reached"
            }

        current_plan = prep_res["current_plan"]
        tasks = prep_res["tasks"]

        if not current_plan:
            return {"action": "truly_complete", "reason": "No active plan"}

        # Check actual completion status
        pending_tasks = [t for t in current_plan.tasks if tasks[t.id].status == "pending"]
        running_tasks = [t for t in current_plan.tasks if tasks[t.id].status == "running"]
        completed_tasks = [t for t in current_plan.tasks if tasks[t.id].status == "completed"]
        failed_tasks = [t for t in current_plan.tasks if tasks[t.id].status == "failed"]

        total_tasks = len(current_plan.tasks)

        # Truly complete: all tasks done
        if len(completed_tasks) + len(failed_tasks) == total_tasks:
            if len(failed_tasks) == 0 or len(completed_tasks) > len(failed_tasks):
                return {"action": "truly_complete", "reason": "All tasks completed"}
            else:
                return {"action": "truly_complete", "reason": "Plan failed but cannot continue"}

        # Has pending tasks that can run
        if pending_tasks and not running_tasks:
            return {"action": "continue_execution", "reason": f"{len(pending_tasks)} tasks ready"}

        # Has running tasks, wait
        if running_tasks:
            return {"action": "continue_execution", "reason": f"{len(running_tasks)} tasks running"}

        # Need reflection if tasks are stuck
        if pending_tasks and not running_tasks:
            return {"action": "needs_reflection", "reason": "Tasks may be blocked"}

        # Default: we're done
        return {"action": "truly_complete", "reason": "No actionable tasks"}

    async def post_async(self, shared, prep_res, exec_res):
        action = exec_res["action"]

        # Reset counter on true completion
        if action == "truly_complete":
            self.execution_count = 0
            shared["flow_completion_reason"] = exec_res["reason"]
        elif action == "force_terminate":  # HINZUGEFÜGT
            self.execution_count = 0
            shared["flow_completion_reason"] = f"Force terminated: {exec_res['reason']}"
            shared["force_terminated"] = True
            wprint(f"Flow force terminated: {exec_res['reason']}")

        return action

ContextAggregatorNode

Bases: AsyncNode

Vereinfachte Context-Aggregation über UnifiedContextManager

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

@with_progress_tracking
class ContextAggregatorNode(AsyncNode):
    """Vereinfachte Context-Aggregation über UnifiedContextManager"""

    async def prep_async(self, shared):
        """Simplified preparation - delegate to UnifiedContextManager"""
        return {
            "context_manager": shared.get("context_manager"),
            "session_id": shared.get("session_id", "default"),
            "original_query": shared.get("current_query", ""),
            "tasks": shared.get("tasks", {}),
            "current_plan": shared.get("current_plan"),
            "world_model": shared.get("world_model", {}),
            "results": shared.get("results", {})
        }

    async def exec_async(self, prep_res):
        """VEREINFACHT: Get aggregated context from UnifiedContextManager"""

        context_manager = prep_res.get("context_manager")
        session_id = prep_res.get("session_id", "default")
        query = prep_res.get("original_query", "")

        if not context_manager:
            # Fallback: Create basic aggregated context
            return self._create_fallback_context(prep_res)

        try:
            #Get unified context from context manager
            unified_context = await context_manager.build_unified_context(session_id, query, "full")

            # Transform to expected aggregated_context format for compatibility
            aggregated_context = {
                "original_query": query,
                "successful_results": self._extract_successful_results(unified_context),
                "failed_attempts": self._extract_failed_attempts(prep_res["tasks"]),
                "key_discoveries": self._extract_key_discoveries(unified_context),
                "adaptation_summary": self._extract_adaptation_summary(prep_res),
                "confidence_scores": self._calculate_confidence_scores(unified_context),
                "unified_context": unified_context,  # Include full unified context
                "context_source": "unified_context_manager"
            }

            return aggregated_context

        except Exception as e:
            eprint(f"UnifiedContextManager aggregation failed: {e}")
            return self._create_fallback_context(prep_res)

    def _extract_successful_results(self, unified_context: dict[str, Any]) -> dict[str, Any]:
        """Extract successful results from unified context"""
        successful_results = {}

        try:
            # Get from variables context
            variables = unified_context.get("variables", {})
            recent_results = variables.get("recent_results", [])

            for result in recent_results:
                if result.get("success"):
                    task_id = result.get("task_id", f"result_{len(successful_results)}")
                    successful_results[task_id] = {
                        "task_description": f"Task {task_id}",
                        "task_type": "unified_context_result",
                        "result": result.get("preview", ""),
                        "metadata": {
                            "timestamp": result.get("timestamp"),
                            "source": "unified_context"
                        }
                    }

            # Also check execution state for completions
            execution_state = unified_context.get("execution_state", {})
            recent_completions = execution_state.get("recent_completions", [])

            for completion in recent_completions:
                task_id = completion.get("id", f"completion_{len(successful_results)}")
                successful_results[task_id] = {
                    "task_description": completion.get("description", "Completed task"),
                    "task_type": "execution_completion",
                    "result": f"Task completed at {completion.get('completed_at', 'unknown time')}",
                    "metadata": {
                        "completion_time": completion.get("completed_at"),
                        "source": "execution_state"
                    }
                }

            return successful_results

        except Exception as e:
            eprint(f"Error extracting successful results: {e}")
            return {}

    def _extract_failed_attempts(self, tasks: dict) -> dict[str, Any]:
        """Extract failed attempts from tasks (existing functionality)"""
        failed_attempts = {}

        try:
            for task_id, task in tasks.items():
                if task.status == "failed":
                    failed_attempts[task_id] = {
                        "description": task.description,
                        "error": task.error,
                        "retry_count": task.retry_count
                    }
            return failed_attempts
        except:
            return {}

    def _extract_key_discoveries(self, unified_context: dict[str, Any]) -> list[dict[str, Any]]:
        """Extract key discoveries from unified context"""
        discoveries = []

        try:
            # Extract from relevant facts
            relevant_facts = unified_context.get("relevant_facts", [])
            for key, value in relevant_facts[:3]:  # Top 3 facts
                discoveries.append({
                    "discovery": f"Fact discovered: {key}",
                    "confidence": 0.8,  # Default confidence for facts
                    "result": value
                })

            # Extract from successful results
            variables = unified_context.get("variables", {})
            recent_results = variables.get("recent_results", [])

            for result in recent_results[:2]:  # Top 2 results
                if result.get("success"):
                    discoveries.append({
                        "discovery": f"Task result: {result.get('task_id', 'unknown')}",
                        "confidence": 0.7,
                        "result": result.get("preview", "")
                    })

            return discoveries

        except Exception as e:
            eprint(f"Error extracting discoveries: {e}")
            return []

    def _extract_adaptation_summary(self, prep_res: dict) -> str:
        """Extract adaptation summary"""
        try:
            current_plan = prep_res.get("current_plan")
            if current_plan and hasattr(current_plan, 'metadata'):
                adaptations = current_plan.metadata.get("adaptations", 0)
                if adaptations > 0:
                    return f"Plan was adapted {adaptations} times to handle unexpected results."
            return ""
        except:
            return ""

    def _calculate_confidence_scores(self, unified_context: dict[str, Any]) -> dict[str, float]:
        """Calculate confidence scores based on unified context"""
        try:
            scores = {"overall": 0.5}

            # Base confidence on available data
            chat_history = unified_context.get("chat_history", [])
            if chat_history:
                scores["conversation_context"] = min(len(chat_history) / 10, 1.0)

            variables = unified_context.get("variables", {})
            recent_results = variables.get("recent_results", [])
            successful_results = [r for r in recent_results if r.get("success")]

            if recent_results:
                scores["execution_results"] = len(successful_results) / len(recent_results)

            # Calculate overall confidence
            scores["overall"] = sum(scores.values()) / len(scores)

            return scores

        except:
            return {"overall": 0.3}

    def _create_fallback_context(self, prep_res: dict) -> dict[str, Any]:
        """Create fallback context when UnifiedContextManager is unavailable"""
        return {
            "original_query": prep_res.get("original_query", ""),
            "successful_results": {},
            "failed_attempts": self._extract_failed_attempts(prep_res.get("tasks", {})),
            "key_discoveries": [],
            "adaptation_summary": "Fallback context - UnifiedContextManager unavailable",
            "confidence_scores": {"overall": 0.2},
            "context_source": "fallback"
        }

    async def post_async(self, shared, prep_res, exec_res):
        """Store aggregated context for downstream nodes"""
        shared["aggregated_context"] = exec_res

        #Also store unified context reference for other nodes
        if "unified_context" in exec_res:
            shared["unified_context"] = exec_res["unified_context"]

        if exec_res.get("successful_results") or exec_res.get("key_discoveries"):
            return "context_ready"
        else:
            return "no_context"

exec_async(prep_res) async

VEREINFACHT: Get aggregated context from UnifiedContextManager

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

async def exec_async(self, prep_res):
    """VEREINFACHT: Get aggregated context from UnifiedContextManager"""

    context_manager = prep_res.get("context_manager")
    session_id = prep_res.get("session_id", "default")
    query = prep_res.get("original_query", "")

    if not context_manager:
        # Fallback: Create basic aggregated context
        return self._create_fallback_context(prep_res)

    try:
        #Get unified context from context manager
        unified_context = await context_manager.build_unified_context(session_id, query, "full")

        # Transform to expected aggregated_context format for compatibility
        aggregated_context = {
            "original_query": query,
            "successful_results": self._extract_successful_results(unified_context),
            "failed_attempts": self._extract_failed_attempts(prep_res["tasks"]),
            "key_discoveries": self._extract_key_discoveries(unified_context),
            "adaptation_summary": self._extract_adaptation_summary(prep_res),
            "confidence_scores": self._calculate_confidence_scores(unified_context),
            "unified_context": unified_context,  # Include full unified context
            "context_source": "unified_context_manager"
        }

        return aggregated_context

    except Exception as e:
        eprint(f"UnifiedContextManager aggregation failed: {e}")
        return self._create_fallback_context(prep_res)

post_async(shared, prep_res, exec_res) async

Store aggregated context for downstream nodes

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

async def post_async(self, shared, prep_res, exec_res):
    """Store aggregated context for downstream nodes"""
    shared["aggregated_context"] = exec_res

    #Also store unified context reference for other nodes
    if "unified_context" in exec_res:
        shared["unified_context"] = exec_res["unified_context"]

    if exec_res.get("successful_results") or exec_res.get("key_discoveries"):
        return "context_ready"
    else:
        return "no_context"

prep_async(shared) async

Simplified preparation - delegate to UnifiedContextManager

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

async def prep_async(self, shared):
    """Simplified preparation - delegate to UnifiedContextManager"""
    return {
        "context_manager": shared.get("context_manager"),
        "session_id": shared.get("session_id", "default"),
        "original_query": shared.get("current_query", ""),
        "tasks": shared.get("tasks", {}),
        "current_plan": shared.get("current_plan"),
        "world_model": shared.get("world_model", {}),
        "results": shared.get("results", {})
    }

DecisionTask dataclass

Bases: Task

Task für dynamisches Routing

Source code in toolboxv2/mods/isaa/base/Agent/types.py

@dataclass
class DecisionTask(Task):
    """Task für dynamisches Routing"""
    decision_prompt: str = ""  # Kurze Frage an LLM
    routing_map: dict[str, str] = field(default_factory=dict)  # Ergebnis -> nächster Task
    decision_model: str = "fast"  # Welches LLM für Entscheidung

FlowAgent

Production-ready agent system built on PocketFlow

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

class FlowAgent:
    """Production-ready agent system built on PocketFlow """
    def __init__(
        self,
        amd: AgentModelData,
        world_model: dict[str, Any] = None,
        verbose: bool = False,
        enable_pause_resume: bool = True,
        checkpoint_interval: int = 300,  # 5 minutes
        max_parallel_tasks: int = 3,
        progress_callback: callable = None,
        stream:bool=True,
        **kwargs
    ):
        self.amd = amd
        self.stream = stream
        self.world_model = world_model or {}
        self.verbose = verbose
        self.enable_pause_resume = enable_pause_resume
        self.checkpoint_interval = checkpoint_interval
        self.max_parallel_tasks = max_parallel_tasks
        self.progress_tracker = ProgressTracker(progress_callback, agent_name=amd.name)

        # Core state
        self.shared = {
            "world_model": self.world_model,
            "tasks": {},
            "task_plans": {},
            "system_status": "idle",
            "session_data": {},
            "performance_metrics": {},
            "conversation_history": [],
            "available_tools": [],
            "progress_tracker": self.progress_tracker
        }
        self.context_manager = UnifiedContextManager(self)
        self.variable_manager = VariableManager(self.shared["world_model"], self.shared)
        self.context_manager.variable_manager = self.variable_manager# Register default scopes

        self.shared["context_manager"] = self.context_manager
        self.shared["variable_manager"] = self.variable_manager
        # Flows
        self.task_flow = TaskManagementFlow(max_parallel_tasks=self.max_parallel_tasks)
        self.response_flow = ResponseGenerationFlow()

        if hasattr(self.task_flow, 'executor_node'):
            self.task_flow.executor_node.agent_instance = self

        # Agent state
        self.is_running = False
        self.is_paused = False
        self.last_checkpoint = None
        self.checkpoint_data = {}
        self.ac_cost = 0

        # Threading
        self.executor = ThreadPoolExecutor(max_workers=max_parallel_tasks)
        self._shutdown_event = threading.Event()

        # Server components
        self.a2a_server: A2AServer = None
        self.mcp_server: FastMCP = None

        # Enhanced tool registry
        self._tool_registry = {}
        self._tool_capabilities = {}
        self._tool_analysis_cache = {}

        self.active_session = None
        # Tool analysis file path
        self.tool_analysis_file = self._get_tool_analysis_path()

        self._tool_capabilities.update(self._load_tool_analysis())
        if self.amd.budget_manager:
            self.amd.budget_manager.load_data()

        self._setup_variable_scopes()

        rprint(f"FlowAgent initialized: {amd.name}")

    def task_flow_settings(self, max_parallel_tasks: int = 3, max_reasoning_loops: int = 24, max_tool_calls:int = 5):
        self.task_flow.executor_node.max_parallel = max_parallel_tasks
        self.task_flow.llm_reasoner.max_reasoning_loops = max_reasoning_loops
        self.task_flow.llm_tool_node.max_tool_calls = max_tool_calls

    @property
    def progress_callback(self):
        return self.progress_tracker.progress_callback

    @progress_callback.setter
    def progress_callback(self, value):
        self.progress_tracker.progress_callback = value

    def set_progress_callback(self, progress_callback: callable = None):
        self.progress_callback = progress_callback

    async def a_run_llm_completion(self, node_name="FlowAgentLLMCall",task_id="unknown",model_preference="fast", with_context=True, auto_fallbacks=True, **kwargs) -> str:
        if "model" not in kwargs:
            kwargs["model"] = self.amd.fast_llm_model if model_preference == "fast" else self.amd.complex_llm_model

        if not 'stream' in kwargs:
            kwargs['stream'] = self.stream

        llm_start = time.perf_counter()

        if self.progress_tracker:
            await self.progress_tracker.emit_event(ProgressEvent(
                event_type="llm_call",
                node_name=node_name,
                session_id=self.active_session,
                task_id=task_id,
                status=NodeStatus.RUNNING,
                llm_model=kwargs["model"],
                llm_temperature=kwargs.get("temperature", 0.7),
                llm_input=kwargs.get("messages", [{}])[-1].get("content", ""),  # Prompt direkt erfassen
                metadata={
                    "model_preference": kwargs.get("model_preference", "fast")
                }
            ))

        # auto api key addition supports (google, openrouter, openai, anthropic, azure, aws, huggingface, replicate, togetherai, groq)
        if "api_key" not in kwargs:
            # litellm model-prefix apikey mapp
            prefix = kwargs['model'].split("/")[0]
            model_prefix_map = {
                "openrouter": os.getenv("OPENROUTER_API_KEY"),
                "openai": os.getenv("OPENAI_API_KEY"),
                "anthropic": os.getenv("ANTHROPIC_API_KEY"),
                "google": os.getenv("GOOGLE_API_KEY"),
                "azure": os.getenv("AZURE_API_KEY"),
                "huggingface": os.getenv("HUGGINGFACE_API_KEY"),
                "replicate": os.getenv("REPLICATE_API_KEY"),
                "togetherai": os.getenv("TOGETHERAI_API_KEY"),
                "groq": os.getenv("GROQ_API_KEY"),
            }
            kwargs["api_key"] = model_prefix_map.get(prefix)

        if self.active_session and with_context:
            # Add context to fist messages as system message
            context_ = await self.get_context(self.active_session)
            kwargs["messages"] = [{"role": "system", "content": self.amd.get_system_message_with_persona()+'\n\nContext:\n\n'+context_}] + kwargs.get("messages", [])

        # build fallback dict using FALLBACKS_MODELS/PREM and _KEYS

        if auto_fallbacks and 'fallbacks' not in kwargs:
            fallbacks_dict_list = []
            fallbacks = os.getenv("FALLBACKS_MODELS", '').split(',') if model_preference == "fast" else os.getenv(
                "FALLBACKS_MODELS_PREM", '').split(',')
            fallbacks_keys = os.getenv("FALLBACKS_MODELS_KEYS", '').split(
                ',') if model_preference == "fast" else os.getenv(
                "FALLBACKS_MODELS_KEYS_PREM", '').split(',')
            for model, key in zip(fallbacks, fallbacks_keys):
                fallbacks_dict_list.append({"model": model, "api_key": os.getenv(key, kwargs.get("api_key", None))})
            kwargs['fallbacks'] = fallbacks_dict_list
        try:

            if kwargs.get("stream", False):
                kwargs["stream_options"] = {"include_usage": True}

            # detailed informations str
            with Spinner(f"LLM Call {self.amd.name}@{node_name}#{task_id if task_id else model_preference}-{kwargs['model']}"):
                response = await litellm.acompletion(**kwargs)

            if not kwargs.get("stream", False):
                result = response.choices[0].message.content
                usage = response.usage
                input_tokens = usage.prompt_tokens if usage else 0
                output_tokens = usage.completion_tokens if usage else 0
                total_tokens = usage.total_tokens if usage else 0

            else:
                result = ""
                final_chunk = None
                async for chunk in response:
                    content = chunk.choices[0].delta.content or ""
                    result += content
                    if self.progress_tracker and content:
                        await self.progress_tracker.emit_event(ProgressEvent(
                            event_type="llm_stream_chunk",
                            node_name=node_name,
                            task_id=task_id,
                            session_id=self.active_session,
                            status=NodeStatus.RUNNING,
                            # weitere Felder wie model, tokens wenn verfügbar
                            llm_model=kwargs["model"],
                            llm_output=content,
                            # optional: llm_tokens so far? usage in chunk?
                        ))
                    final_chunk = chunk

                usage = final_chunk.usage if hasattr(final_chunk, "usage") else None
                output_tokens = usage.completion_tokens if usage else 0
                input_tokens = usage.prompt_tokens if usage else 0
                total_tokens = usage.total_tokens if usage else 0
            llm_duration = time.perf_counter() - llm_start

            if AGENT_VERBOSE and self.verbose:
                kwargs["messages"] += [{"role": "assistant", "content": result}]
                print_prompt(kwargs)
            # else:
            #     print_prompt([{"role": "assistant", "content": result}])

            # Extract token usage and cost


            call_cost = self.progress_tracker.calculate_llm_cost(kwargs["model"], input_tokens,
                                                            output_tokens, response) if self.progress_tracker else 0.0
            self.ac_cost += call_cost
            if self.progress_tracker:
                await self.progress_tracker.emit_event(ProgressEvent(
                    event_type="llm_call",
                    node_name=node_name,
                    task_id=task_id,
                    session_id=self.active_session,
                    status=NodeStatus.COMPLETED,
                    success=True,
                    duration=llm_duration,
                    llm_model=kwargs["model"],
                    llm_prompt_tokens=input_tokens,
                    llm_completion_tokens=output_tokens,
                    llm_total_tokens=total_tokens,
                    llm_cost=call_cost,
                    llm_temperature=kwargs.get("temperature", 0.7),
                    llm_output=result,
                    llm_input="",
                ))

            return result
        except Exception as e:
            llm_duration = time.perf_counter() - llm_start
            import traceback
            print(traceback.format_exc())
            # print(f"LLM call failed: {json.dumps(kwargs, indent=2)}")

            if self.progress_tracker:
                await self.progress_tracker.emit_event(ProgressEvent(
                    event_type="llm_call",  # Event-Typ bleibt konsistent
                    node_name=node_name,
                    task_id=task_id,
                    session_id=self.active_session,
                    status=NodeStatus.FAILED,
                    success=False,
                    duration=llm_duration,
                    llm_model=kwargs["model"],
                    error_details={
                        "message": str(e),
                        "type": type(e).__name__
                    }
                ))

            raise

    async def a_run(
        self,
        query: str,
        session_id: str = "default",
        user_id: str = None,
        stream_callback: Callable = None,
        remember: bool = True,
        **kwargs
    ) -> str:
        """Main entry point für Agent-Ausführung mit UnifiedContextManager"""

        execution_start = self.progress_tracker.start_timer("total_execution")
        self.active_session = session_id
        result = None
        await self.progress_tracker.emit_event(ProgressEvent(
            event_type="execution_start",
            timestamp=time.time(),
            status=NodeStatus.RUNNING,
            node_name="FlowAgent",
            session_id=session_id,
            metadata={"query": query, "user_id": user_id}
        ))

        try:
            #Initialize or get session über UnifiedContextManager
            await self.initialize_session_context(session_id, max_history=200)

            #Store user message immediately in ChatSession wenn remember=True
            if remember:
                await self.context_manager.add_interaction(
                    session_id,
                    'user',
                    query,
                    metadata={"user_id": user_id}
                )

            # Set user context variables
            timestamp = datetime.now()
            self.variable_manager.register_scope('user', {
                'id': user_id,
                'session': session_id,
                'query': query,
                'timestamp': timestamp.isoformat()
            })

            # Update system variables
            self.variable_manager.set('system_context.timestamp', {'isoformat': timestamp.isoformat()})
            self.variable_manager.set('system_context.current_session', session_id)
            self.variable_manager.set('system_context.current_user', user_id)
            self.variable_manager.set('system_context.last_query', query)

            # Initialize with tool awareness
            await self.initialize_context_awareness()

            # VEREINFACHT: Prepare execution context - weniger Daten duplizieren
            self.shared.update({
                "current_query": query,
                "session_id": session_id,
                "user_id": user_id,
                "stream_callback": stream_callback,
                "remember": remember,
                # CENTRAL: Context Manager ist die primäre Context-Quelle
                "context_manager": self.context_manager,
                "variable_manager": self.variable_manager
            })

            # Set LLM models in shared context
            self.shared['fast_llm_model'] = self.amd.fast_llm_model
            self.shared['complex_llm_model'] = self.amd.complex_llm_model
            self.shared['persona_config'] = self.amd.persona
            self.shared['use_fast_response'] = self.amd.use_fast_response

            # Set system status
            self.shared["system_status"] = "running"
            self.is_running = True

            # Execute main orchestration flow
            result = await self._orchestrate_execution()

            #Store assistant response in ChatSession wenn remember=True
            if remember:
                await self.context_manager.add_interaction(
                    session_id,
                    'assistant',
                    result,
                    metadata={"user_id": user_id, "execution_duration": time.time() - execution_start}
                )

            total_duration = self.progress_tracker.end_timer("total_execution")

            await self.progress_tracker.emit_event(ProgressEvent(
                event_type="execution_complete",
                timestamp=time.time(),
                node_name="FlowAgent",
                status=NodeStatus.COMPLETED,
                node_duration=total_duration,
                session_id=session_id,
                metadata={
                    "result_length": len(result),
                    "summary": self.progress_tracker.get_summary(),
                    "remembered": remember
                }
            ))

            # Checkpoint if needed
            if self.enable_pause_resume:
                with Spinner("Creating checkpoint..."):
                    await self._maybe_checkpoint()
            return result

        except Exception as e:
            eprint(f"Agent execution failed: {e}", exc_info=True)
            error_response = f"I encountered an error: {str(e)}"
            result = error_response
            import traceback
            print(traceback.format_exc())

            # Store error in ChatSession wenn remember=True
            if remember:
                await self.context_manager.add_interaction(
                    session_id,
                    'assistant',
                    error_response,
                    metadata={
                        "user_id": user_id,
                        "error": True,
                        "error_type": type(e).__name__
                    }
                )

            total_duration = self.progress_tracker.end_timer("total_execution")

            await self.progress_tracker.emit_event(ProgressEvent(
                event_type="error",
                timestamp=time.time(),
                node_name="FlowAgent",
                status=NodeStatus.FAILED,
                node_duration=total_duration,
                session_id=session_id,
                metadata={"error": str(e), "error_type": type(e).__name__}
            ))

            return error_response

        finally:
            self.shared["system_status"] = "idle"
            self.is_running = False
            self.active_session = None

    def set_response_format(
        self,
        response_format: str,
        text_length: str,
        custom_instructions: str = "",
        quality_threshold: float = 0.7
    ):
        """Dynamische Format- und Längen-Konfiguration"""

        # Validiere Eingaben
        try:
            ResponseFormat(response_format)
            TextLength(text_length)
        except ValueError:
            available_formats = [f.value for f in ResponseFormat]
            available_lengths = [l.value for l in TextLength]
            raise ValueError(
                f"Invalid format or length. "
                f"Available formats: {available_formats}. "
                f"Available lengths: {available_lengths}"
            )

        # Erstelle oder aktualisiere Persona
        if not self.amd.persona:
            self.amd.persona = PersonaConfig(name="Assistant")

        # Erstelle Format-Konfiguration
        format_config = FormatConfig(
            response_format=ResponseFormat(response_format),
            text_length=TextLength(text_length),
            custom_instructions=custom_instructions,
            quality_threshold=quality_threshold
        )

        self.amd.persona.format_config = format_config

        # Aktualisiere Personality Traits mit Format-Hinweisen
        self._update_persona_with_format(response_format, text_length)

        # Update shared state
        self.shared["persona_config"] = self.amd.persona
        self.shared["format_config"] = format_config

        rprint(f"Response format set: {response_format}, length: {text_length}")

    def _update_persona_with_format(self, response_format: str, text_length: str):
        """Aktualisiere Persona-Traits basierend auf Format"""

        # Format-spezifische Traits
        format_traits = {
            "with-tables": ["structured", "data-oriented", "analytical"],
            "with-bullet-points": ["organized", "clear", "systematic"],
            "with-lists": ["methodical", "sequential", "thorough"],
            "md-text": ["technical", "formatted", "detailed"],
            "yaml-text": ["structured", "machine-readable", "precise"],
            "json-text": ["technical", "API-focused", "structured"],
            "text-only": ["conversational", "natural", "flowing"],
            "pseudo-code": ["logical", "algorithmic", "step-by-step"],
            "code-structure": ["technical", "systematic", "hierarchical"]
        }

        # Längen-spezifische Traits
        length_traits = {
            "mini-chat": ["concise", "quick", "to-the-point"],
            "chat-conversation": ["conversational", "friendly", "balanced"],
            "table-conversation": ["structured", "comparative", "organized"],
            "detailed-indepth": ["thorough", "comprehensive", "analytical"],
            "phd-level": ["academic", "scholarly", "authoritative"]
        }

        # Kombiniere Traits
        current_traits = set(self.amd.persona.personality_traits)

        # Entferne alte Format-Traits
        old_format_traits = set()
        for traits in format_traits.values():
            old_format_traits.update(traits)
        for traits in length_traits.values():
            old_format_traits.update(traits)

        current_traits -= old_format_traits

        # Füge neue Traits hinzu
        new_traits = format_traits.get(response_format, [])
        new_traits.extend(length_traits.get(text_length, []))

        current_traits.update(new_traits)
        self.amd.persona.personality_traits = list(current_traits)

    def get_available_formats(self) -> dict[str, list[str]]:
        """Erhalte verfügbare Format- und Längen-Optionen"""
        return {
            "formats": [f.value for f in ResponseFormat],
            "lengths": [l.value for l in TextLength],
            "format_descriptions": {
                f.value: FormatConfig(response_format=f).get_format_instructions()
                for f in ResponseFormat
            },
            "length_descriptions": {
                l.value: FormatConfig(text_length=l).get_length_instructions()
                for l in TextLength
            }
        }

    async def a_run_with_format(
        self,
        query: str,
        response_format: str = "frei-text",
        text_length: str = "chat-conversation",
        custom_instructions: str = "",
        **kwargs
    ) -> str:
        """Führe Agent mit spezifischem Format aus"""

        # Temporäre Format-Einstellung
        original_persona = self.amd.persona

        try:
            self.set_response_format(response_format, text_length, custom_instructions)
            response = await self.a_run(query, **kwargs)
            return response
        finally:
            # Restore original persona
            self.amd.persona = original_persona
            self.shared["persona_config"] = original_persona

    def get_format_quality_report(self) -> dict[str, Any]:
        """Erhalte detaillierten Format-Qualitätsbericht"""
        quality_assessment = self.shared.get("quality_assessment", {})

        if not quality_assessment:
            return {"status": "no_assessment", "message": "No recent quality assessment available"}

        quality_details = quality_assessment.get("quality_details", {})

        return {
            "overall_score": quality_details.get("total_score", 0.0),
            "format_adherence": quality_details.get("format_adherence", 0.0),
            "length_adherence": quality_details.get("length_adherence", 0.0),
            "content_quality": quality_details.get("base_quality", 0.0),
            "llm_assessment": quality_details.get("llm_assessment", 0.0),
            "suggestions": quality_assessment.get("suggestions", []),
            "assessment": quality_assessment.get("quality_assessment", "unknown"),
            "format_config_active": quality_details.get("format_config_used", False)
        }

    def get_variable_documentation(self) -> str:
        """Get comprehensive variable system documentation"""
        docs = []
        docs.append("# Variable System Documentation\n")

        # Available scopes
        docs.append("## Available Scopes:")
        scope_info = self.variable_manager.get_scope_info()
        for scope_name, info in scope_info.items():
            docs.append(f"- `{scope_name}`: {info['type']} with {info.get('keys', 'N/A')} keys")

        docs.append("\n## Syntax Options:")
        docs.append("- `{{ variable.path }}` - Full path resolution")
        docs.append("- `{variable}` - Simple variable (no dots)")
        docs.append("- `$variable` - Shell-style variable")

        docs.append("\n## Example Usage:")
        docs.append("- `{{ results.task_1.data }}` - Get result from task_1")
        docs.append("- `{{ user.name }}` - Get user name")
        docs.append("- `{agent_name}` - Simple agent name")
        docs.append("- `$timestamp` - System timestamp")

        # Available variables
        docs.append("\n## Available Variables:")
        variables = self.variable_manager.get_available_variables()
        for scope_name, scope_vars in variables.items():
            docs.append(f"\n### {scope_name}:")
            for _var_name, var_info in scope_vars.items():
                docs.append(f"- `{var_info['path']}`: {var_info['preview']} ({var_info['type']})")

        return "\n".join(docs)

    def _setup_variable_scopes(self):
        """Setup default variable scopes with enhanced structure"""
        self.variable_manager.register_scope('agent', {
            'name': self.amd.name,
            'model_fast': self.amd.fast_llm_model,
            'model_complex': self.amd.complex_llm_model
        })

        timestamp = datetime.now()
        self.variable_manager.register_scope('system', {
            'timestamp': timestamp.isoformat(),
            'version': '2.0',
            'capabilities': list(self._tool_capabilities.keys())
        })

        # ADDED: Initialize empty results and tasks scopes
        self.variable_manager.register_scope('results', {})
        self.variable_manager.register_scope('tasks', {})

        # Update shared state
        self.shared["variable_manager"] = self.variable_manager

    def set_variable(self, path: str, value: Any):
        """Set variable using unified system"""
        self.variable_manager.set(path, value)

    def get_variable(self, path: str, default=None):
        """Get variable using unified system"""
        return self.variable_manager.get(path, default)

    def format_text(self, text: str, **context) -> str:
        """Format text with variables"""
        return self.variable_manager.format_text(text, context)

    async def initialize_session_context(self, session_id: str = "default", max_history: int = 200) -> bool:
        """Vereinfachte Session-Initialisierung über UnifiedContextManager"""
        try:
            # Delegation an UnifiedContextManager
            session = await self.context_manager.initialize_session(session_id, max_history)

            # Ensure Variable Manager integration
            if not self.context_manager.variable_manager:
                self.context_manager.variable_manager = self.variable_manager

            # Update shared state (minimal - primary data now in context_manager)
            self.shared["active_session_id"] = session_id
            self.shared["session_initialized"] = True

            # Legacy support: Keep session_managers reference in shared for backward compatibility
            self.shared["session_managers"] = self.context_manager.session_managers

            rprint(f"Session context initialized for {session_id} via UnifiedContextManager")
            return True

        except Exception as e:
            eprint(f"Session context initialization failed: {e}")
            import traceback
            print(traceback.format_exc())
            return False

    async def initialize_context_awareness(self):
        """Enhanced context awareness with session management"""

        # Initialize session if not already done
        session_id = self.shared.get("session_id", self.active_session)
        if not self.shared.get("session_initialized"):
            await self.initialize_session_context(session_id)

        # Ensure tool capabilities are loaded
        # add tqdm prigress bar

        from tqdm import tqdm

        if hasattr(self.task_flow, 'llm_reasoner'):
            if "read_from_variables" not in self.shared["available_tools"] and hasattr(self.task_flow.llm_reasoner, '_execute_read_from_variables'):
                await self.add_tool(lambda scope, key, purpose: self.task_flow.llm_reasoner._execute_read_from_variables({"scope": scope, "key": key, "purpose": purpose}), "read_from_variables", "Read from variables")
            if "write_to_variables" not in self.shared["available_tools"] and hasattr(self.task_flow.llm_reasoner, '_execute_write_to_variables'):
                await self.add_tool(lambda scope, key, value, description: self.task_flow.llm_reasoner._execute_write_to_variables({"scope": scope, "key": key, "value": value, "description": description}), "write_to_variables", "Write to variables")

            if "internal_reasoning" not in self.shared["available_tools"] and hasattr(self.task_flow.llm_reasoner, '_execute_internal_reasoning'):
                async def internal_reasoning_tool(thought:str, thought_number:int, total_thoughts:int, next_thought_needed:bool, current_focus:str, key_insights:list[str], potential_issues:list[str], confidence_level:float):
                    args = {
                        "thought": thought,
                        "thought_number": thought_number,
                        "total_thoughts": total_thoughts,
                        "next_thought_needed": next_thought_needed,
                        "current_focus": current_focus,
                        "key_insights": key_insights,
                        "potential_issues": potential_issues,
                        "confidence_level": confidence_level
                    }
                    return await self.task_flow.llm_reasoner._execute_internal_reasoning(args, self.shared)
                await self.add_tool(internal_reasoning_tool, "internal_reasoning", "Internal reasoning")

            if "manage_internal_task_stack" not in self.shared["available_tools"] and hasattr(self.task_flow.llm_reasoner, '_execute_manage_task_stack'):
                async def manage_internal_task_stack_tool(action:str, task_description:str, outline_step_ref:str):
                    args = {
                        "action": action,
                        "task_description": task_description,
                        "outline_step_ref": outline_step_ref
                    }
                    return await self.task_flow.llm_reasoner._execute_manage_task_stack(args, self.shared)
                await self.add_tool(manage_internal_task_stack_tool, "manage_internal_task_stack", "Manage internal task stack")

            if "outline_step_completion" not in self.shared["available_tools"] and hasattr(self.task_flow.llm_reasoner, '_execute_outline_step_completion'):
                async def outline_step_completion_tool(step_completed:bool, completion_evidence:str, next_step_focus:str):
                    args = {
                        "step_completed": step_completed,
                        "completion_evidence": completion_evidence,
                        "next_step_focus": next_step_focus
                    }
                    return await self.task_flow.llm_reasoner._execute_outline_step_completion(args, self.shared)
                await self.add_tool(outline_step_completion_tool, "outline_step_completion", "Outline step completion")


        registered_tools = set(self._tool_registry.keys())
        cached_capabilities = list(self._tool_capabilities.keys())  # Create a copy of
        for tool_name in cached_capabilities:
            if tool_name in self._tool_capabilities and tool_name not in registered_tools:
                del self._tool_capabilities[tool_name]
                iprint(f"Removed outdated capability for unavailable tool: {tool_name}")

        for tool_name in tqdm(self.shared["available_tools"], desc=f"Agent {self.amd.name} Analyzing Tools", unit="tool", colour="green", total=len(self.shared["available_tools"])):
            if tool_name not in self._tool_capabilities:
                tool_info = self._tool_registry.get(tool_name, {})
                description = tool_info.get("description", "No description")
                with Spinner(f"Analyzing tool {tool_name}"):
                    await self._analyze_tool_capabilities(tool_name, description, tool_info.get("args_schema", "()"))

            if tool_name in self._tool_capabilities:
                function = self._tool_registry[tool_name]["function"]
                if not isinstance(self._tool_capabilities[tool_name], dict):
                    self._tool_capabilities[tool_name] = {}
                self._tool_capabilities[tool_name]["args_schema"] = get_args_schema(function)

        # Set enhanced system context
        self.shared["system_context"] = {
            "capabilities_summary": self._build_capabilities_summary(),
            "tool_count": len(self.shared["available_tools"]),
            "analysis_loaded": len(self._tool_capabilities),
            "intelligence_level": "high" if self._tool_capabilities else "basic",
            "context_management": "advanced_session_aware",
            "session_managers": len(self.shared.get("session_managers", {})),
        }


        rprint("Advanced context awareness initialized with session management")

    async def get_context(self, session_id: str = None, format_for_llm: bool = True) -> str | dict[str, Any]:
        """
        ÜBERARBEITET: Get context über UnifiedContextManager statt verteilte Quellen
        """
        try:
            session_id = session_id or self.shared.get("session_id", self.active_session)
            query = self.shared.get("current_query", "")

            #Hole unified context über Context Manager
            unified_context = await self.context_manager.build_unified_context(session_id, query, "full")


            if format_for_llm:
                return self.context_manager.get_formatted_context_for_llm(unified_context)
            else:
                return unified_context

        except Exception as e:
            import traceback
            print(traceback.format_exc())
            eprint(f"Failed to generate context via UnifiedContextManager: {e}")

            # FALLBACK: Fallback zu alter Methode falls UnifiedContextManager fehlschlägt
            if format_for_llm:
                return f"Error generating context: {str(e)}"
            else:
                return {
                    "error": str(e),
                    "generated_at": datetime.now().isoformat(),
                    "fallback_mode": True
                }

    def get_context_statistics(self) -> dict[str, Any]:
        """Get comprehensive context management statistics"""
        stats = {
            "context_system": "advanced_session_aware",
            "compression_threshold": 0.76,
            "max_tokens": getattr(self, 'max_input_tokens', 8000),
            "session_managers": {},
            "context_usage": {},
            "compression_stats": {}
        }

        # Session manager statistics
        session_managers = self.shared.get("session_managers", {})
        for name, manager in session_managers.items():
            stats["session_managers"][name] = {
                "history_length": len(manager.history if hasattr(manager, 'history') else (manager.get("history", []) if hasattr(manager, 'get') else [])),
                "max_length": manager.max_length if hasattr(manager, 'max_length') else manager.get("max_length", 0),
                "space_name": manager.space_name if hasattr(manager, 'space_name') else manager.get("space_name", "")
            }

        # Context node statistics if available
        if hasattr(self.task_flow, 'context_manager'):
            context_manager = self.task_flow.context_manager
            stats["compression_stats"] = {
                "compression_threshold": context_manager.compression_threshold,
                "max_tokens": context_manager.max_tokens,
                "active_sessions": len(context_manager.session_managers)
            }

        # LLM call statistics from enhanced node
        llm_stats = self.shared.get("llm_call_stats", {})
        if llm_stats:
            stats["context_usage"] = {
                "total_llm_calls": llm_stats.get("total_calls", 0),
                "context_compression_rate": llm_stats.get("context_compression_rate", 0.0),
                "average_context_tokens": llm_stats.get("context_tokens_used", 0) / max(llm_stats.get("total_calls", 1),
                                                                                        1)
            }

        return stats

    def set_persona(self, name: str, style: str = "professional", tone: str = "friendly",
                    personality_traits: list[str] = None, apply_method: str = "system_prompt",
                    integration_level: str = "light", custom_instructions: str = ""):
        """Set agent persona mit erweiterten Konfigurationsmöglichkeiten"""
        if personality_traits is None:
            personality_traits = ["helpful", "concise"]

        self.amd.persona = PersonaConfig(
            name=name,
            style=style,
            tone=tone,
            personality_traits=personality_traits,
            custom_instructions=custom_instructions,
            apply_method=apply_method,
            integration_level=integration_level
        )

        rprint(f"Persona set: {name} ({style}, {tone}) - Method: {apply_method}, Level: {integration_level}")

    def configure_persona_integration(self, apply_method: str = "system_prompt", integration_level: str = "light"):
        """Configure how persona is applied"""
        if self.amd.persona:
            self.amd.persona.apply_method = apply_method
            self.amd.persona.integration_level = integration_level
            rprint(f"Persona integration updated: {apply_method}, {integration_level}")
        else:
            wprint("No persona configured to update")

    def get_available_variables(self) -> dict[str, str]:
        """Get available variables for dynamic formatting"""
        return self.variable_manager.get_available_variables()

    async def _orchestrate_execution(self) -> str:
        """
        Enhanced orchestration with LLMReasonerNode as strategic core.
        The reasoner now handles both task management and response generation internally.
        """

        self.shared["agent_instance"] = self
        self.shared["session_id"] = self.active_session
        # === UNIFIED REASONING AND EXECUTION CYCLE ===
        rprint("Starting strategic reasoning and execution cycle")

        # The LLMReasonerNode now handles the complete cycle:
        # 1. Strategic analysis of the query
        # 2. Decision making about approach
        # 3. Orchestration of sub-systems (LLMToolNode, TaskPlanner/Executor)
        # 4. Response synthesis and formatting

        # Execute the unified flow
        task_management_result = await self.task_flow.run_async(self.shared)

        # Check for various completion states
        if self.shared.get("plan_halted"):
            error_response = f"Task execution was halted: {self.shared.get('halt_reason', 'Unknown reason')}"
            self.shared["current_response"] = error_response
            return error_response

        final_response = self.shared.get("current_response", "Task completed successfully.")
        # Execute ResponseGenerationFlow for persona application and formatting
        response_result = await self.response_flow.run_async(self.shared)

        # The reasoner provides the final response
        final_response = self.shared.get("current_response", "Task completed successfully.")

        # Add reasoning artifacts to response if available
        reasoning_artifacts = self.shared.get("reasoning_artifacts", {})
        if reasoning_artifacts and reasoning_artifacts.get("reasoning_loops", 0) > 1:
            # For debugging/transparency, could add reasoning info to metadata
            pass

        # Log enhanced statistics
        self._log_execution_stats()

        return final_response

    def _log_execution_stats(self):
        """Enhanced execution statistics with reasoning metrics"""
        tasks = self.shared.get("tasks", {})
        adaptations = self.shared.get("plan_adaptations", 0)
        reasoning_artifacts = self.shared.get("reasoning_artifacts", {})

        completed_tasks = sum(1 for t in tasks.values() if t.status == "completed")
        failed_tasks = sum(1 for t in tasks.values() if t.status == "failed")

        # Enhanced logging with reasoning metrics
        reasoning_loops = reasoning_artifacts.get("reasoning_loops", 0)

        stats_message = f"Execution complete - Tasks: {completed_tasks} completed, {failed_tasks} failed"

        if adaptations > 0:
            stats_message += f", {adaptations} adaptations"

        if reasoning_loops > 0:
            stats_message += f", {reasoning_loops} reasoning loops"

            # Add reasoning efficiency metric
            if completed_tasks > 0:
                efficiency = completed_tasks / max(reasoning_loops, 1)
                stats_message += f" (efficiency: {efficiency:.1f} tasks/loop)"

        rprint(stats_message)

        # Log reasoning context if significant
        if reasoning_loops > 3:
            internal_task_stack = reasoning_artifacts.get("internal_task_stack", [])
            completed_reasoning_tasks = len([t for t in internal_task_stack if t.get("status") == "completed"])

            if completed_reasoning_tasks > 0:
                rprint(f"Strategic reasoning: {completed_reasoning_tasks} high-level tasks completed")

    def _build_capabilities_summary(self) -> str:
        """Build summary of agent capabilities"""

        if not self._tool_capabilities:
            return "Basic LLM capabilities only"

        summaries = []
        for tool_name, cap in self._tool_capabilities.items():
            primary = cap.get('primary_function', 'Unknown function')
            summaries.append(f"{tool_name}{cap.get('args_schema', '()')}: {primary}")

        return f"Enhanced capabilities: {'; '.join(summaries)}"

    # Neue Hilfsmethoden für erweiterte Funktionalität

    async def get_task_execution_summary(self) -> dict[str, Any]:
        """Erhalte detaillierte Zusammenfassung der Task-Ausführung"""
        tasks = self.shared.get("tasks", {})
        results_store = self.shared.get("results", {})

        summary = {
            "total_tasks": len(tasks),
            "completed_tasks": [],
            "failed_tasks": [],
            "task_types_used": {},
            "tools_used": [],
            "adaptations": self.shared.get("plan_adaptations", 0),
            "execution_timeline": []
        }

        for task_id, task in tasks.items():
            task_info = {
                "id": task_id,
                "type": task.type,
                "description": task.description,
                "status": task.status,
                "duration": None
            }

            if task.started_at and task.completed_at:
                duration = (task.completed_at - task.started_at).total_seconds()
                task_info["duration"] = duration

            if task.status == "completed":
                summary["completed_tasks"].append(task_info)
                if isinstance(task, ToolTask):
                    summary["tools_used"].append(task.tool_name)
            elif task.status == "failed":
                task_info["error"] = task.error
                summary["failed_tasks"].append(task_info)

            # Task types counting
            task_type = task.type
            summary["task_types_used"][task_type] = summary["task_types_used"].get(task_type, 0) + 1

        return summary

    async def explain_reasoning_process(self) -> str:
        """Erkläre den Reasoning-Prozess des Agenten"""
        if not LITELLM_AVAILABLE:
            return "Reasoning explanation requires LLM capabilities."

        summary = await self.get_task_execution_summary()

        prompt = f"""
Erkläre den Reasoning-Prozess dieses AI-Agenten in verständlicher Form:

## Ausführungszusammenfassung
- Total Tasks: {summary['total_tasks']}
- Erfolgreich: {len(summary['completed_tasks'])}
- Fehlgeschlagen: {len(summary['failed_tasks'])}
- Plan-Adaptationen: {summary['adaptations']}
- Verwendete Tools: {', '.join(set(summary['tools_used']))}
- Task-Typen: {summary['task_types_used']}

## Task-Details
Erfolgreiche Tasks:
{self._format_tasks_for_explanation(summary['completed_tasks'])}

## Anweisungen
Erkläre in 2-3 Absätzen:
1. Welche Strategie der Agent gewählt hat
2. Wie er die Aufgabe in Tasks unterteilt hat
3. Wie er auf unerwartete Ergebnisse reagiert hat (falls Adaptationen)
4. Was die wichtigsten Erkenntnisse waren

Schreibe für einen technischen Nutzer, aber verständlich."""

        try:
            response = await self.a_run_llm_completion(
                model=self.amd.complex_llm_model,
                messages=[{"role": "user", "content": prompt}],
                temperature=0.5,
                max_tokens=800,task_id="reasoning_explanation"
            )

            return response

        except Exception as e:
            import traceback
            print(traceback.format_exc())
            return f"Could not generate reasoning explanation: {e}"

    def _format_tasks_for_explanation(self, tasks: list[dict]) -> str:
        formatted = []
        for task in tasks[:5]:  # Top 5 tasks
            duration_info = f" ({task['duration']:.1f}s)" if task['duration'] else ""
            formatted.append(f"- {task['type']}: {task['description']}{duration_info}")
        return "\n".join(formatted)

    # ===== PAUSE/RESUME FUNCTIONALITY =====

    async def pause(self) -> bool:
        """Pause agent execution"""
        if not self.is_running:
            return False

        self.is_paused = True
        self.shared["system_status"] = "paused"

        # Create checkpoint
        checkpoint = await self._create_checkpoint()
        await self._save_checkpoint(checkpoint)

        rprint("Agent execution paused")
        return True

    async def resume(self) -> bool:
        """Resume agent execution"""
        if not self.is_paused:
            return False

        self.is_paused = False
        self.shared["system_status"] = "running"

        rprint("Agent execution resumed")
        return True

    # ===== CHECKPOINT MANAGEMENT =====

    async def _create_checkpoint(self) -> AgentCheckpoint:
        """Vereinfachte Checkpoint-Erstellung - fokussiert auf wesentliche Daten"""
        try:
            # Budget Manager Daten vor Checkpoint speichern
            if hasattr(self.amd, 'budget_manager') and self.amd.budget_manager:
                self.amd.budget_manager.save_data()

            # Bereite AMD-Daten vor (ohne budget_manager für Serialisierung)
            amd_data = self.amd.model_dump()
            amd_data['budget_manager'] = None

            # Sammle wesentliche Session-Daten (vereinfacht)
            session_data = {}
            if self.context_manager and self.context_manager.session_managers:
                for session_id, session in self.context_manager.session_managers.items():
                    try:
                        if hasattr(session, 'history') and session.history:
                            # Nur die letzten 20 Nachrichten pro Session für Checkpoint
                            recent_history = session.history[-20:]
                            session_data[session_id] = {
                                "history": recent_history,
                                "session_type": "chatsession",
                                "message_count": len(session.history)
                            }
                        elif isinstance(session, dict) and session.get('history'):
                            session_data[session_id] = {
                                "history": session['history'][-20:],
                                "session_type": "fallback",
                                "message_count": len(session['history'])
                            }
                    except Exception as e:
                        rprint(f"Skipping session {session_id} in checkpoint: {e}")

            # Sammle serialisierbare Variable-Scopes
            variable_scopes = {}
            if self.variable_manager:
                NON_SERIALIZABLE_KEYS = {
                    "tool_registry", "variable_manager", "context_manager", "agent_instance",
                    "llm_tool_node_instance", "task_planner_instance", "task_executor_instance",
                    "progress_tracker", "session_managers", "stream_callback"
                }

                for scope_name, scope_data in self.variable_manager.scopes.items():
                    if isinstance(scope_data, dict):
                        # Filtere nicht-serialisierbare Objekte heraus
                        clean_scope = {
                            k: v for k, v in scope_data.items()
                            if k not in NON_SERIALIZABLE_KEYS
                        }
                        variable_scopes[scope_name] = clean_scope
                    else:
                        try:
                            # Teste Serialisierbarkeit
                            pickle.dumps(scope_data)
                            variable_scopes[scope_name] = scope_data
                        except:
                            # Überspringe nicht-serialisierbare Scopes
                            pass

            # Erstelle konsolidierten Checkpoint
            checkpoint = AgentCheckpoint(
                timestamp=datetime.now(),
                agent_state={
                    "is_running": self.is_running,
                    "is_paused": self.is_paused,
                    "amd_data": amd_data,
                    "active_session": self.active_session,
                    "system_status": self.shared.get("system_status", "idle")
                },
                task_state={
                    task_id: asdict(task) for task_id, task in self.shared.get("tasks", {}).items()
                },
                world_model=self.shared.get("world_model", {}),
                active_flows=["task_flow", "response_flow"],
                metadata={
                    "session_id": self.shared.get("session_id", "default"),
                    "last_query": self.shared.get("current_query", ""),
                    "checkpoint_version": "3.0_simplified",
                    "agent_name": self.amd.name
                },
                # Konsolidierte Zusatzdaten
                session_data=session_data,
                variable_scopes=variable_scopes,
                results_store=self.shared.get("results", {}),
                conversation_history=self.shared.get("conversation_history", [])[-50:],  # Letzte 50 Nachrichten
                tool_capabilities=self._tool_capabilities.copy()
            )

            rprint(f"Vereinfachter Checkpoint erstellt mit {len(session_data)} Sessions")
            return checkpoint

        except Exception as e:
            eprint(f"Checkpoint-Erstellung fehlgeschlagen: {e}")
            import traceback
            print(traceback.format_exc())
            raise

    async def _save_checkpoint(self, checkpoint: AgentCheckpoint, filepath: str = None):
        """Vereinfachtes Checkpoint-Speichern - alles in eine Datei"""
        try:
            from toolboxv2 import get_app
            folder = str(get_app().data_dir) + '/Agents/checkpoint/' + self.amd.name
            if not os.path.exists(folder):
                os.makedirs(folder, exist_ok=True)

            if not filepath:
                timestamp = checkpoint.timestamp.strftime("%Y%m%d_%H%M%S")
                filepath = f"agent_checkpoint_{timestamp}.pkl"
            filepath = os.path.join(folder, filepath)

            # Sessions vor dem Speichern synchronisieren
            if self.context_manager and self.context_manager.session_managers:
                for session_id, session in self.context_manager.session_managers.items():
                    try:
                        if hasattr(session, 'save'):
                            await session.save()
                        elif hasattr(session, '_save_to_memory'):
                            session._save_to_memory()
                    except Exception as e:
                        rprint(f"Session sync error für {session_id}: {e}")

            # Speichere Checkpoint direkt
            with open(filepath, 'wb') as f:
                pickle.dump(checkpoint, f)

            self.last_checkpoint = checkpoint.timestamp

            # Erstelle einfache Zusammenfassung
            summary_parts = []
            if hasattr(checkpoint, 'session_data') and checkpoint.session_data:
                summary_parts.append(f"{len(checkpoint.session_data)} sessions")
            if checkpoint.task_state:
                completed_tasks = len([t for t in checkpoint.task_state.values() if t.get("status") == "completed"])
                summary_parts.append(f"{completed_tasks} completed tasks")
            if hasattr(checkpoint, 'variable_scopes') and checkpoint.variable_scopes:
                summary_parts.append(f"{len(checkpoint.variable_scopes)} variable scopes")

            summary = "; ".join(summary_parts) if summary_parts else "Basic checkpoint"
            rprint(f"Checkpoint gespeichert: {filepath} ({summary})")
            return True

        except Exception as e:
            eprint(f"Checkpoint-Speicherung fehlgeschlagen: {e}")
            import traceback
            print(traceback.format_exc())
            return False

    async def load_latest_checkpoint(self, auto_restore_history: bool = True, max_age_hours: int = 24) -> dict[
        str, Any]:
        """Vereinfachtes Checkpoint-Laden mit automatischer History-Wiederherstellung"""
        try:
            from toolboxv2 import get_app
            folder = str(get_app().data_dir) + '/Agents/checkpoint/' + self.amd.name

            if not os.path.exists(folder):
                return {"success": False, "error": "Kein Checkpoint-Verzeichnis gefunden"}

            # Finde neuesten Checkpoint
            checkpoint_files = []
            for file in os.listdir(folder):
                if file.endswith('.pkl') and file.startswith('agent_checkpoint_'):
                    filepath = os.path.join(folder, file)
                    try:
                        timestamp_str = file.replace('agent_checkpoint_', '').replace('.pkl', '')
                        if timestamp_str == 'final_checkpoint':
                            file_time = datetime.fromtimestamp(os.path.getmtime(filepath))
                        else:
                            file_time = datetime.strptime(timestamp_str, "%Y%m%d_%H%M%S")

                        age_hours = (datetime.now() - file_time).total_seconds() / 3600
                        if age_hours <= max_age_hours:
                            checkpoint_files.append((filepath, file_time, age_hours))
                    except Exception:
                        continue

            if not checkpoint_files:
                return {"success": False, "error": f"Keine gültigen Checkpoints in {max_age_hours} Stunden gefunden"}

            # Lade neuesten Checkpoint
            checkpoint_files.sort(key=lambda x: x[1], reverse=True)
            latest_checkpoint_path, latest_timestamp, age_hours = checkpoint_files[0]

            rprint(f"Lade Checkpoint: {latest_checkpoint_path} (Alter: {age_hours:.1f}h)")

            with open(latest_checkpoint_path, 'rb') as f:
                checkpoint: AgentCheckpoint = pickle.load(f)

            # Stelle Agent-Status wieder her
            restore_stats = await self._restore_from_checkpoint_simplified(checkpoint, auto_restore_history)

            # Re-initialisiere Kontext-Awareness
            await self.initialize_context_awareness()

            return {
                "success": True,
                "checkpoint_file": latest_checkpoint_path,
                "checkpoint_age_hours": age_hours,
                "checkpoint_timestamp": latest_timestamp.isoformat(),
                "available_checkpoints": len(checkpoint_files),
                "restore_stats": restore_stats
            }

        except Exception as e:
            eprint(f"Checkpoint-Laden fehlgeschlagen: {e}")
            import traceback
            print(traceback.format_exc())
            return {"success": False, "error": str(e)}

    async def _restore_from_checkpoint_simplified(self, checkpoint: AgentCheckpoint, auto_restore_history: bool) -> \
    dict[
        str, Any]:
        """Vereinfachte Checkpoint-Wiederherstellung"""
        restore_stats = {
            "agent_state_restored": False,
            "world_model_restored": False,
            "tasks_restored": 0,
            "sessions_restored": 0,
            "variables_restored": 0,
            "conversation_restored": 0,
            "errors": []
        }

        try:
            # 1. Agent-Status wiederherstellen
            if checkpoint.agent_state:
                self.is_running = checkpoint.agent_state.get("is_running", False)
                self.is_paused = checkpoint.agent_state.get("is_paused", False)
                self.active_session = checkpoint.agent_state.get("active_session")

                # AMD-Daten selektiv wiederherstellen
                amd_data = checkpoint.agent_state.get("amd_data", {})
                if amd_data:
                    # Nur sichere Felder wiederherstellen
                    safe_fields = ["name", "use_fast_response", "max_input_tokens"]
                    for field in safe_fields:
                        if field in amd_data and hasattr(self.amd, field):
                            setattr(self.amd, field, amd_data[field])

                    # Persona wiederherstellen falls vorhanden
                    if "persona" in amd_data and amd_data["persona"]:
                        try:
                            persona_data = amd_data["persona"]
                            if isinstance(persona_data, dict):
                                self.amd.persona = PersonaConfig(**persona_data)
                        except Exception as e:
                            restore_stats["errors"].append(f"Persona restore failed: {e}")

                restore_stats["agent_state_restored"] = True

            # 2. World Model wiederherstellen
            if checkpoint.world_model:
                self.shared["world_model"] = checkpoint.world_model.copy()
                self.world_model = checkpoint.world_model.copy()
                restore_stats["world_model_restored"] = True

            # 3. Variable System wiederherstellen
            if hasattr(checkpoint, 'variable_scopes') and checkpoint.variable_scopes:
                # Variable Manager neu initialisieren
                self.variable_manager = VariableManager(self.shared["world_model"], self.shared)

                # Basis-Scopes einrichten
                self._setup_variable_scopes()

                # Gespeicherte Scopes wiederherstellen
                for scope_name, scope_data in checkpoint.variable_scopes.items():
                    try:
                        self.variable_manager.register_scope(scope_name, scope_data)
                        restore_stats["variables_restored"] += 1
                    except Exception as e:
                        restore_stats["errors"].append(f"Variable scope {scope_name}: {e}")

                # Runtime-Objekte wieder einsetzen
                self.variable_manager.set("shared", "variable_manager", self.variable_manager)
                self.variable_manager.set("shared", "context_manager", self.context_manager)
                self.variable_manager.set("shared", "agent_instance", self)

                self.shared["variable_manager"] = self.variable_manager

            # 4. Tasks wiederherstellen
            if checkpoint.task_state:
                restored_tasks = {}
                for task_id, task_data in checkpoint.task_state.items():
                    try:
                        task_type = task_data.get("type", "generic")
                        if task_type == "LLMTask":
                            restored_tasks[task_id] = LLMTask(**task_data)
                        elif task_type == "ToolTask":
                            restored_tasks[task_id] = ToolTask(**task_data)
                        elif task_type == "DecisionTask":
                            restored_tasks[task_id] = DecisionTask(**task_data)
                        else:
                            restored_tasks[task_id] = Task(**task_data)

                        restore_stats["tasks_restored"] += 1
                    except Exception as e:
                        restore_stats["errors"].append(f"Task {task_id}: {e}")

                self.shared["tasks"] = restored_tasks

            # 5. Results Store wiederherstellen
            if hasattr(checkpoint, 'results_store') and checkpoint.results_store:
                self.shared["results"] = checkpoint.results_store
                if self.variable_manager:
                    self.variable_manager.set_results_store(checkpoint.results_store)

            # 6. Sessions und Conversation wiederherstellen (falls gewünscht)
            if auto_restore_history:
                await self._restore_sessions_and_conversation_simplified(checkpoint, restore_stats)

            # 7. Tool Capabilities wiederherstellen
            if hasattr(checkpoint, 'tool_capabilities') and checkpoint.tool_capabilities:
                self._tool_capabilities = checkpoint.tool_capabilities.copy()

            # Status setzen
            self.shared["system_status"] = "restored"
            restore_stats["restoration_timestamp"] = datetime.now().isoformat()

            rprint(
                f"Checkpoint wiederhergestellt: {restore_stats['tasks_restored']} Tasks, {restore_stats['sessions_restored']} Sessions, {len(restore_stats['errors'])} Fehler")
            return restore_stats

        except Exception as e:
            eprint(f"Checkpoint-Wiederherstellung fehlgeschlagen: {e}")
            import traceback
            print(traceback.format_exc())
            restore_stats["errors"].append(f"Critical restore error: {e}")
            return restore_stats

    async def _restore_sessions_and_conversation_simplified(self, checkpoint: AgentCheckpoint, restore_stats: dict):
        """Vereinfachte Session- und Conversation-Wiederherstellung"""
        try:
            # Context Manager sicherstellen
            if not self.context_manager:
                self.context_manager = UnifiedContextManager(self)
                self.context_manager.variable_manager = self.variable_manager

            # Sessions wiederherstellen
            if hasattr(checkpoint, 'session_data') and checkpoint.session_data:
                for session_id, session_info in checkpoint.session_data.items():
                    try:
                        # Session über Context Manager initialisieren
                        max_length = session_info.get("message_count", 200)
                        restored_session = await self.context_manager.initialize_session(session_id, max_length)

                        # History wiederherstellen
                        history = session_info.get("history", [])
                        if history and hasattr(restored_session, 'history'):
                            # Direkt in Session-History einfügen
                            restored_session.history.extend(history)

                        restore_stats["sessions_restored"] += 1
                    except Exception as e:
                        restore_stats["errors"].append(f"Session {session_id}: {e}")

            # Conversation History wiederherstellen
            if hasattr(checkpoint, 'conversation_history') and checkpoint.conversation_history:
                self.shared["conversation_history"] = checkpoint.conversation_history
                restore_stats["conversation_restored"] = len(checkpoint.conversation_history)

            # Update shared context
            self.shared["context_manager"] = self.context_manager
            if self.context_manager.session_managers:
                self.shared["session_managers"] = self.context_manager.session_managers
                self.shared["session_initialized"] = True

        except Exception as e:
            restore_stats["errors"].append(f"Session/conversation restore failed: {e}")

    async def _maybe_checkpoint(self):
        """Vereinfachtes automatisches Checkpointing"""
        if not self.enable_pause_resume:
            return

        now = datetime.now()
        if (not self.last_checkpoint or
            (now - self.last_checkpoint).seconds >= self.checkpoint_interval):

            try:
                checkpoint = await self._create_checkpoint()
                await self._save_checkpoint(checkpoint)
            except Exception as e:
                eprint(f"Automatic checkpoint failed: {e}")


    async def save_context_to_session(self, session_id: str = None, context_type: str = "full") -> bool:
        """Save current context to ChatSession for persistent storage"""
        try:
            session_id = session_id or self.shared.get("session_id", "default")

            if not self.context_manager:
                eprint("Context manager not available")
                return False

            # Build comprehensive context
            unified_context = await self.context_manager.build_unified_context(session_id, None, context_type)

            # Create context message for session storage
            context_message = {
                "role": "system",
                "content": f"[CONTEXT_SNAPSHOT_{context_type.upper()}] " + json.dumps(unified_context, default=str),
                "timestamp": datetime.now().isoformat(),
                "context_type": context_type,
                "metadata": {
                    "is_context_snapshot": True,
                    "context_version": "2.0",
                    "agent_name": self.amd.name,
                    "session_stats": unified_context.get("session_stats", {}),
                    "variables_count": len(unified_context.get("variables", {}).get("recent_results", [])),
                    "execution_state": unified_context.get("execution_state", {}).get("system_status", "unknown")
                }
            }

            # Store in session
            await self.context_manager.add_interaction(
                session_id,
                "system",
                context_message["content"],
                metadata=context_message["metadata"]
            )

            rprint(f"Context snapshot saved to session {session_id} (type: {context_type})")
            return True

        except Exception as e:
            eprint(f"Failed to save context to session: {e}")
            return False

    async def load_context_from_session(self, session_id: str, context_type: str = "full") -> dict[str, Any]:
        """Load context from ChatSession storage"""
        try:
            if not self.context_manager:
                return {"error": "Context manager not available"}

            session = self.context_manager.session_managers.get(session_id)
            if not session:
                return {"error": f"Session {session_id} not found"}

            # Search for context snapshots in session history
            context_snapshots = []

            if hasattr(session, 'history'):
                for message in reversed(session.history):  # Search from newest
                    if (message.get("role") == "system" and
                        message.get("metadata", {}).get("is_context_snapshot") and
                        message.get("metadata", {}).get("context_type") == context_type):

                        try:
                            # Extract context data
                            content = message.get("content", "")
                            if content.startswith(f"[CONTEXT_SNAPSHOT_{context_type.upper()}]"):
                                json_data = content.replace(f"[CONTEXT_SNAPSHOT_{context_type.upper()}] ", "")
                                context_data = json.loads(json_data)
                                context_snapshots.append({
                                    "context": context_data,
                                    "timestamp": message.get("timestamp"),
                                    "metadata": message.get("metadata", {})
                                })
                        except Exception as e:
                            wprint(f"Failed to parse context snapshot: {e}")

            if context_snapshots:
                # Return most recent context snapshot
                latest_context = context_snapshots[0]
                rprint(f"Loaded context snapshot from session {session_id} (timestamp: {latest_context['timestamp']})")
                return latest_context["context"]
            else:
                return {"error": f"No context snapshots of type '{context_type}' found in session {session_id}"}

        except Exception as e:
            eprint(f"Failed to load context from session: {e}")
            return {"error": str(e)}

    async def cleanup_session_context(self, session_id: str = None, keep_count: int = 100,
                                      remove_old_snapshots: bool = True) -> dict[str, Any]:
        """Cleanup session context by removing old snapshots and entries"""
        try:
            session_id = session_id or self.shared.get("session_id", "default")

            if not self.context_manager:
                return {"error": "Context manager not available"}

            session = self.context_manager.session_managers.get(session_id)
            if not session or not hasattr(session, 'history'):
                return {"error": f"Session {session_id} not found or has no history"}

            cleanup_stats = {
                "original_message_count": len(session.history),
                "context_snapshots_removed": 0,
                "context_entries_removed": 0,
                "regular_messages_kept": 0,
                "cleanup_performed": False
            }

            if len(session.history) <= keep_count:
                return {**cleanup_stats, "message": "No cleanup needed"}

            # Separate different types of messages
            regular_messages = []
            context_snapshots = []
            context_entries = []

            for message in session.history:
                metadata = message.get("metadata", {})

                if metadata.get("is_context_snapshot"):
                    context_snapshots.append(message)
                elif metadata.get("is_context_entry"):
                    context_entries.append(message)
                else:
                    regular_messages.append(message)

            # Keep most recent regular messages
            messages_to_keep = regular_messages[-keep_count:]
            cleanup_stats["regular_messages_kept"] = len(messages_to_keep)

            # Keep most recent context snapshots (if not removing)
            if not remove_old_snapshots:
                recent_snapshots = context_snapshots[-5:]  # Keep last 5 snapshots
                messages_to_keep.extend(recent_snapshots)
            else:
                cleanup_stats["context_snapshots_removed"] = len(context_snapshots)

            # Keep persistent context entries
            persistent_entries = [
                entry for entry in context_entries
                if entry.get("persistent", True)
            ]
            messages_to_keep.extend(persistent_entries)
            cleanup_stats["context_entries_removed"] = len(context_entries) - len(persistent_entries)

            # Sort by timestamp and update session
            messages_to_keep.sort(key=lambda x: x.get("timestamp", ""))
            session.history = messages_to_keep

            cleanup_stats.update({
                "final_message_count": len(session.history),
                "cleanup_performed": True,
                "messages_removed": cleanup_stats["original_message_count"] - len(session.history)
            })

            rprint(f"Session cleanup completed: {cleanup_stats['messages_removed']} messages removed")
            return cleanup_stats

        except Exception as e:
            eprint(f"Failed to cleanup session context: {e}")
            return {"error": str(e)}

    def get_session_storage_stats(self) -> dict[str, Any]:
        """Get comprehensive session storage statistics"""
        try:
            stats = {
                "context_manager_active": bool(self.context_manager),
                "total_sessions": 0,
                "session_details": {},
                "storage_summary": {
                    "total_messages": 0,
                    "context_snapshots": 0,
                    "context_entries": 0,
                    "regular_messages": 0
                }
            }

            if not self.context_manager:
                return stats

            stats["total_sessions"] = len(self.context_manager.session_managers)

            for session_id, session in self.context_manager.session_managers.items():
                session_stats = {
                    "session_type": "chatsession" if hasattr(session, 'history') else "fallback",
                    "message_count": 0,
                    "context_snapshots": 0,
                    "context_entries": 0,
                    "regular_messages": 0,
                    "storage_size_estimate": 0
                }

                if hasattr(session, 'history'):
                    session_stats["message_count"] = len(session.history)

                    for message in session.history:
                        content_size = len(str(message))
                        session_stats["storage_size_estimate"] += content_size

                        metadata = message.get("metadata", {})
                        if metadata.get("is_context_snapshot"):
                            session_stats["context_snapshots"] += 1
                        elif metadata.get("is_context_entry"):
                            session_stats["context_entries"] += 1
                        else:
                            session_stats["regular_messages"] += 1

                elif isinstance(session, dict) and 'history' in session:
                    session_stats["message_count"] = len(session['history'])
                    session_stats["regular_messages"] = len(session['history'])
                    session_stats["storage_size_estimate"] = sum(len(str(msg)) for msg in session['history'])

                stats["session_details"][session_id] = session_stats

                # Update totals
                stats["storage_summary"]["total_messages"] += session_stats["message_count"]
                stats["storage_summary"]["context_snapshots"] += session_stats["context_snapshots"]
                stats["storage_summary"]["context_entries"] += session_stats["context_entries"]
                stats["storage_summary"]["regular_messages"] += session_stats["regular_messages"]

            # Estimate total storage size
            stats["storage_summary"]["estimated_total_size_kb"] = sum(
                details["storage_size_estimate"] for details in stats["session_details"].values()
            ) / 1024

            return stats

        except Exception as e:
            eprint(f"Failed to get session storage stats: {e}")
            return {"error": str(e)}

    def list_available_checkpoints(self, max_age_hours: int = 168) -> list[dict[str, Any]]:  # Default 1 week
        """List all available checkpoints with metadata"""
        try:
            from toolboxv2 import get_app
            folder = str(get_app().data_dir) + '/Agents/checkpoint/' + self.amd.name

            if not os.path.exists(folder):
                return []

            checkpoints = []
            for file in os.listdir(folder):
                if file.endswith('.pkl') and file.startswith('agent_checkpoint_'):
                    filepath = os.path.join(folder, file)
                    try:
                        # Get file info
                        file_stat = os.stat(filepath)
                        file_size = file_stat.st_size
                        modified_time = datetime.fromtimestamp(file_stat.st_mtime)

                        # Extract timestamp from filename
                        timestamp_str = file.replace('agent_checkpoint_', '').replace('.pkl', '')
                        if timestamp_str == 'final_checkpoint':
                            checkpoint_time = modified_time
                            checkpoint_type = "final"
                        else:
                            checkpoint_time = datetime.strptime(timestamp_str, "%Y%m%d_%H%M%S")
                            checkpoint_type = "regular"

                        # Check age
                        age_hours = (datetime.now() - checkpoint_time).total_seconds() / 3600
                        if age_hours <= max_age_hours:

                            # Try to load checkpoint metadata without full loading
                            metadata = {}
                            try:
                                with open(filepath, 'rb') as f:
                                    checkpoint = pickle.load(f)
                                metadata = {
                                    "tasks_count": len(checkpoint.task_state) if checkpoint.task_state else 0,
                                    "world_model_entries": len(checkpoint.world_model) if checkpoint.world_model else 0,
                                    "session_id": checkpoint.metadata.get("session_id", "unknown") if hasattr(
                                        checkpoint, 'metadata') and checkpoint.metadata else "unknown",
                                    "last_query": checkpoint.metadata.get("last_query", "unknown")[:100] if hasattr(
                                        checkpoint, 'metadata') and checkpoint.metadata else "unknown"
                                }
                            except:
                                metadata = {"load_error": True}

                            checkpoints.append({
                                "filepath": filepath,
                                "filename": file,
                                "checkpoint_type": checkpoint_type,
                                "timestamp": checkpoint_time.isoformat(),
                                "age_hours": round(age_hours, 1),
                                "file_size_kb": round(file_size / 1024, 1),
                                "metadata": metadata
                            })

                    except Exception as e:
                        import traceback
                        print(traceback.format_exc())
                        wprint(f"Could not analyze checkpoint file {file}: {e}")
                        continue

            # Sort by timestamp (newest first)
            checkpoints.sort(key=lambda x: x["timestamp"], reverse=True)

            return checkpoints

        except Exception as e:
            import traceback
            print(traceback.format_exc())
            eprint(f"Failed to list checkpoints: {e}")
            return []

    async def delete_old_checkpoints(self, keep_count: int = 5, max_age_hours: int = 168) -> dict[str, Any]:
        """Delete old checkpoints, keeping the most recent ones"""
        try:
            checkpoints = self.list_available_checkpoints(
                max_age_hours=max_age_hours * 2)  # Look further back for deletion

            deleted_count = 0
            deleted_size_kb = 0
            errors = []

            if len(checkpoints) > keep_count:
                # Keep the newest, delete the rest (except final checkpoint)
                to_delete = checkpoints[keep_count:]

                for checkpoint in to_delete:
                    if checkpoint["checkpoint_type"] != "final":  # Never delete final checkpoint
                        try:
                            os.remove(checkpoint["filepath"])
                            deleted_count += 1
                            deleted_size_kb += checkpoint["file_size_kb"]
                            rprint(f"Deleted old checkpoint: {checkpoint['filename']}")
                        except Exception as e:
                            import traceback
                            print(traceback.format_exc())
                            errors.append(f"Failed to delete {checkpoint['filename']}: {e}")

            # Also delete checkpoints older than max_age_hours
            old_checkpoints = [cp for cp in checkpoints if
                               cp["age_hours"] > max_age_hours and cp["checkpoint_type"] != "final"]
            for checkpoint in old_checkpoints:
                if checkpoint not in checkpoints[keep_count:]:  # Don't double-delete
                    try:
                        os.remove(checkpoint["filepath"])
                        deleted_count += 1
                        deleted_size_kb += checkpoint["file_size_kb"]
                        rprint(f"Deleted aged checkpoint: {checkpoint['filename']}")
                    except Exception as e:
                        import traceback
                        print(traceback.format_exc())
                        errors.append(f"Failed to delete {checkpoint['filename']}: {e}")

            return {
                "success": True,
                "deleted_count": deleted_count,
                "freed_space_kb": round(deleted_size_kb, 1),
                "remaining_checkpoints": len(checkpoints) - deleted_count,
                "errors": errors
            }

        except Exception as e:
            import traceback
            print(traceback.format_exc())
            eprint(f"Failed to delete old checkpoints: {e}")
            return {
                "success": False,
                "error": str(e),
                "deleted_count": 0
            }

    # ===== TOOL AND NODE MANAGEMENT =====
    def _get_tool_analysis_path(self) -> str:
        """Get path for tool analysis cache"""
        from toolboxv2 import get_app
        folder = str(get_app().data_dir) + '/Agents/capabilities/' + self.amd.name
        os.makedirs(folder, exist_ok=True)
        return folder + '/tool_capabilities.json'

    def _get_context_path(self, session_id=None) -> str:
        """Get path for tool analysis cache"""
        from toolboxv2 import get_app
        folder = str(get_app().data_dir) + '/Agents/context/' + self.amd.name
        os.makedirs(folder, exist_ok=True)
        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
        session_suffix = f"_session_{session_id}" if session_id else ""
        filepath = f"agent_context_{self.amd.name}_{timestamp}{session_suffix}.json"
        return folder + f'/{filepath}'

    def add_first_class_tool(self, tool_func: Callable, name: str, description: str):
        """
        Add a first-class meta-tool that can be used by the LLMReasonerNode.
        These are different from regular tools - they control agent sub-systems.

        Args:
            tool_func: The function to register as a meta-tool
            name: Name of the meta-tool
            description: Description of when and how to use it
        """

        if not asyncio.iscoroutinefunction(tool_func):
            @wraps(tool_func)
            async def async_wrapper(*args, **kwargs):
                return await asyncio.to_thread(tool_func, *args, **kwargs)

            effective_func = async_wrapper
        else:
            effective_func = tool_func

        tool_name = name or effective_func.__name__
        tool_description = description or effective_func.__doc__ or "No description"

        # Validate the tool function
        if not callable(tool_func):
            raise ValueError("Tool function must be callable")

        # Register in the reasoner's meta-tool registry (if reasoner exists)
        if hasattr(self.task_flow, 'llm_reasoner'):
            if not hasattr(self.task_flow.llm_reasoner, 'meta_tools_registry'):
                self.task_flow.llm_reasoner.meta_tools_registry = {}

            self.task_flow.llm_reasoner.meta_tools_registry[tool_name] = {
                "function": effective_func,
                "description": tool_description,
                "args_schema": get_args_schema(tool_func)
            }

            rprint(f"First-class meta-tool added: {tool_name}")
        else:
            wprint("LLMReasonerNode not available for first-class tool registration")

    async def add_tool(self, tool_func: Callable, name: str = None, description: str = None, is_new=False):
        """Enhanced tool addition with intelligent analysis"""
        if not asyncio.iscoroutinefunction(tool_func):
            @wraps(tool_func)
            async def async_wrapper(*args, **kwargs):
                return await asyncio.to_thread(tool_func, *args, **kwargs)

            effective_func = async_wrapper
        else:
            effective_func = tool_func

        tool_name = name or effective_func.__name__
        tool_description = description or effective_func.__doc__ or "No description"

        # Store in registry
        self._tool_registry[tool_name] = {
            "function": effective_func,
            "description": tool_description,
            "args_schema": get_args_schema(tool_func)
        }

        # Add to available tools list
        if tool_name not in self.shared["available_tools"]:
            self.shared["available_tools"].append(tool_name)

        # Intelligent tool analysis
        if is_new:
            await self._analyze_tool_capabilities(tool_name, tool_description)

        rprint(f"Tool added with analysis: {tool_name}")

    async def _analyze_tool_capabilities(self, tool_name: str, description: str, tool_args:str):
        """Analyze tool capabilities with LLM for smart usage"""

        # Try to load existing analysis
        existing_analysis = self._load_tool_analysis()

        if tool_name in existing_analysis:
            try:
                # Validate cached data against the Pydantic model
                ToolAnalysis.model_validate(existing_analysis[tool_name])
                self._tool_capabilities[tool_name] = existing_analysis[tool_name]
                rprint(f"Loaded and validated cached analysis for {tool_name}")
            except ValidationError as e:
                wprint(f"Cached data for {tool_name} is invalid and will be regenerated: {e}")
                del self._tool_capabilities[tool_name]

        if not LITELLM_AVAILABLE:
            # Fallback analysis
            self._tool_capabilities[tool_name] = {
                "use_cases": [description],
                "triggers": [tool_name.lower().replace('_', ' ')],
                "complexity": "unknown",
                "confidence": 0.3
            }
            return

        # LLM-based intelligent analysis
        prompt = f"""
Analyze this tool and identify ALL possible use cases, triggers, and connections:

Tool Name: {tool_name}
args: {tool_args}
Description: {description}


Provide a comprehensive analysis covering:

1. OBVIOUS use cases (direct functionality)
2. INDIRECT connections (when this tool might be relevant)
3. TRIGGER PHRASES (what user queries would benefit from this tool)
4. COMPLEX scenarios (non-obvious applications)
5. CONTEXTUAL usage (when combined with other information)

Example for a "get_user_name" tool:
- Obvious: When user asks "what is my name"
- Indirect: Personalization, greetings, user identification
- Triggers: "my name", "who am I", "hello", "introduce yourself", "personalize"
- Complex: User context in multi-step tasks, addressing user directly
- Contextual: Any response that could be personalized

Rule! no additional comments or text in the format !
schema:
 {yaml.dump(ToolAnalysis.model_json_schema())}

Respond in YAML format:
Example:
```yaml
primary_function: "Retrieves the current user's name."
use_cases:
  - "Responding to 'what is my name?'"
  - "Personalizing greeting messages."
trigger_phrases:
  - "my name"
  - "who am I"
  - "introduce yourself"
indirect_connections:
  - "User identification in multi-factor authentication."
  - "Tagging user-generated content."
complexity_scenarios:
  - "In a multi-step task, remembering the user's name to personalize the final output."
user_intent_categories:
  - "Personalization"
  - "User Identification"
confidence_triggers:
  "my name": 0.95
  "who am I": 0.9
tool_complexity: low/medium/high
```
"""
        model = os.getenv("BASEMODEL", self.amd.fast_llm_model)
        for i in range(3):
            try:
                response = await self.a_run_llm_completion(
                    model=model,
                    messages=[{"role": "user", "content": prompt}],
                    with_context=False,
                    temperature=0.3,
                    max_tokens=1000,
                    task_id=f"tool_analysis_{tool_name}"
                )

                content = response.strip()

                # Extract JSON
                if "```yaml" in content:
                    yaml_str = content.split("```yaml")[1].split("```")[0].strip()
                else:
                    yaml_str = content

                analysis = yaml.safe_load(yaml_str)

                # Store analysis
                self._tool_capabilities[tool_name] = analysis

                # Save to cache
                await self._save_tool_analysis()

                validated_analysis = ToolAnalysis.model_validate(analysis)
                rprint(f"Generated intelligent analysis for {tool_name}")
                break

            except Exception as e:
                import traceback
                print(traceback.format_exc())
                model = self.amd.complex_llm_model if i > 1 else self.amd.fast_llm_model
                eprint(f"Tool analysis failed for {tool_name}: {e}")
                # Fallback
                self._tool_capabilities[tool_name] = {
                    "primary_function": description,
                    "use_cases": [description],
                    "trigger_phrases": [tool_name.lower().replace('_', ' ')],
                    "tool_complexity": "medium"
                }

    def _load_tool_analysis(self) -> dict[str, Any]:
        """Load tool analysis from cache"""
        try:
            if os.path.exists(self.tool_analysis_file):
                with open(self.tool_analysis_file) as f:
                    return json.load(f)
        except Exception as e:
            wprint(f"Could not load tool analysis: {e}")
        return {}


    async def save_context_to_file(self, session_id: str = None) -> bool:
        """Save current context to file"""
        try:
            context = await self.get_context(session_id=session_id, format_for_llm=False)

            filepath = self._get_context_path(session_id)

            with open(filepath, 'w', encoding='utf-8') as f:
                json.dump(context, f, indent=2, ensure_ascii=False, default=str)

            rprint(f"Context saved to: {filepath}")
            return True

        except Exception as e:
            eprint(f"Failed to save context: {e}")
            return False

    async def _save_tool_analysis(self):
        """Save tool analysis to cache"""
        try:
            with open(self.tool_analysis_file, 'w') as f:
                json.dump(self._tool_capabilities, f, indent=2)
        except Exception as e:
            eprint(f"Could not save tool analysis: {e}")

    def add_custom_flow(self, flow: AsyncFlow, name: str):
        """Add a custom flow for dynamic execution"""
        self.add_tool(flow.run_async, name=name, description=f"Custom flow: {flow.__class__.__name__}")
        rprint(f"Custom node added: {name}")

    def get_tool_by_name(self, tool_name: str) -> Callable | None:
        """Get tool function by name"""
        return self._tool_registry.get(tool_name, {}).get("function")

    async def arun_function(self, function_name: str, *args, **kwargs) -> Any:
        """
        Asynchronously finds a function by its string name, executes it with
        the given arguments, and returns the result.
        """
        rprint(f"Attempting to run function: {function_name} with args: {args}, kwargs: {kwargs}")
        target_function = self.get_tool_by_name(function_name)

        start_time = time.perf_counter()
        if not target_function:
            raise ValueError(f"Function '{function_name}' not found in the {self.amd.name}'s registered tools.")

        try:
            if asyncio.iscoroutinefunction(target_function):
                result = await target_function(*args, **kwargs)
            else:
                # If the function is not async, run it in a thread pool
                loop = asyncio.get_running_loop()
                result = await loop.run_in_executor(None, lambda: target_function(*args, **kwargs))

            if asyncio.iscoroutine(result):
                result = await result

            if self.progress_tracker:
                await self.progress_tracker.emit_event(ProgressEvent(
                    event_type="tool_call",  # Vereinheitlicht zu tool_call
                    node_name="FlowAgent",
                    status=NodeStatus.COMPLETED,
                    success=True,
                    duration=time.perf_counter() - start_time,
                    tool_name=function_name,
                    tool_args=kwargs,
                    tool_result=result,
                    is_meta_tool=False,  # Klarstellen, dass es kein Meta-Tool ist
                    metadata={
                        "result_type": type(result).__name__,
                        "result_length": len(str(result))
                    }
                ))
            rprint(f"Function {function_name} completed successfully with result: {result}")
            return result

        except Exception as e:
            eprint(f"Function {function_name} execution failed: {e}")
            raise

    # ===== FORMATTING =====

    async def a_format_class(self,
                             pydantic_model: type[BaseModel],
                             prompt: str,
                             message_context: list[dict] = None,
                             max_retries: int = 2, auto_context=True, session_id: str = None, **kwargs) -> dict[str, Any]:
        """
        State-of-the-art LLM-based structured data formatting using Pydantic models.

        Args:
            pydantic_model: The Pydantic model class to structure the response
            prompt: The main prompt for the LLM
            message_context: Optional conversation context messages
            max_retries: Maximum number of retry attempts

        Returns:
            dict: Validated structured data matching the Pydantic model

        Raises:
            ValidationError: If the LLM response cannot be validated against the model
            RuntimeError: If all retry attempts fail
        """

        if not LITELLM_AVAILABLE:
            raise RuntimeError("LiteLLM is required for structured formatting but not available")

        if session_id and self.active_session != session_id:
            self.active_session = session_id
        # Generate schema documentation
        schema = pydantic_model.model_json_schema() if issubclass(pydantic_model, BaseModel) else (json.loads(pydantic_model) if isinstance(pydantic_model, str) else pydantic_model)
        model_name = pydantic_model.__name__ if hasattr(pydantic_model, "__name__") else (pydantic_model.get("title", "UnknownModel") if isinstance(pydantic_model, dict) else "UnknownModel")

        # Create enhanced prompt with schema
        enhanced_prompt = f"""
    {prompt}

    CRITICAL FORMATTING REQUIREMENTS:
    1. Respond ONLY in valid YAML format
    2. Follow the exact schema structure provided
    3. Use appropriate data types (strings, lists, numbers, booleans)
    4. Include ALL required fields
    5. No additional comments, explanations, or text outside the YAML

    SCHEMA FOR {model_name}:
    {yaml.dump(schema, default_flow_style=False, indent=2)}

    EXAMPLE OUTPUT FORMAT:
    ```yaml
    # Your response here following the schema exactly
    field_name: "value"
    list_field:
      - "item1"
      - "item2"
    boolean_field: true
    number_field: 42
Respond in YAML format only:
"""
        # Prepare messages
        messages = []
        if message_context:
            messages.extend(message_context)
        messages.append({"role": "user", "content": enhanced_prompt})

        # Retry logic with progressive adjustments
        last_error = None

        for attempt in range(max_retries + 1):
            try:
                # Adjust parameters based on attempt
                temperature = 0.1 + (attempt * 0.1)  # Increase temperature slightly on retries
                max_tokens = min(2000 + (attempt * 500), 4000)  # Increase token limit on retries

                rprint(f"[{model_name}] Attempt {attempt + 1}/{max_retries + 1} (temp: {temperature})")

                # Generate LLM response
                response = await self.a_run_llm_completion(
                    model=self.amd.complex_llm_model,
                    messages=messages,
                    stream=False,
                    with_context=auto_context,
                    temperature=temperature,
                    max_tokens=max_tokens,
                    task_id=f"format_{model_name.lower()}_{attempt}"
                )

                if not response or not response.strip():
                    raise ValueError("Empty response from LLM")

                # Extract YAML content with multiple fallback strategies

                yaml_content = self._extract_yaml_content(response)


                if not yaml_content:
                    raise ValueError("No valid YAML content found in response")

                # Parse YAML
                try:
                    parsed_data = yaml.safe_load(yaml_content)
                except yaml.YAMLError as e:
                    raise ValueError(f"Invalid YAML syntax: {e}")
                iprint(parsed_data)
                if not isinstance(parsed_data, dict):
                    raise ValueError(f"Expected dict, got {type(parsed_data)}")

                # Validate against Pydantic model
                try:
                    if isinstance(pydantic_model, BaseModel):
                        validated_instance = pydantic_model.model_validate(parsed_data)
                        validated_data = validated_instance.model_dump()
                    else:
                        validated_data = parsed_data

                    rprint(f"✅ Successfully formatted {model_name} on attempt {attempt + 1}")
                    return validated_data

                except ValidationError as e:
                    detailed_errors = []
                    for error in e.errors():
                        field_path = " -> ".join(str(x) for x in error['loc'])
                        detailed_errors.append(f"Field '{field_path}': {error['msg']}")

                    error_msg = "Validation failed:\n" + "\n".join(detailed_errors)
                    raise ValueError(error_msg)

            except Exception as e:
                last_error = e
                wprint(f"[{model_name}] Attempt {attempt + 1} failed: {str(e)}")

                if attempt < max_retries:
                    # Add error feedback for next attempt
                    error_feedback = f"\n\nPREVIOUS ATTEMPT FAILED: {str(e)}\nPlease correct the issues and provide valid YAML matching the schema exactly."
                    messages[-1]["content"] = enhanced_prompt + error_feedback

                    # Brief delay before retry
                    # await asyncio.sleep(0.5 * (attempt + 1))
                else:
                    eprint(f"[{model_name}] All {max_retries + 1} attempts failed")

        # All attempts failed
        raise RuntimeError(f"Failed to format {model_name} after {max_retries + 1} attempts. Last error: {last_error}")

    def _extract_yaml_content(self, response: str) -> str:
        """Extract YAML content from LLM response with multiple strategies"""
        # Strategy 1: Extract from code blocks
        if "```yaml" in response:
            try:
                yaml_content = response.split("```yaml")[1].split("```")[0].strip()
                if yaml_content:
                    return yaml_content
            except IndexError:
                pass

        # Strategy 2: Extract from generic code blocks
        if "```" in response:
            try:
                parts = response.split("```")
                for i, part in enumerate(parts):
                    if i % 2 == 1:  # Odd indices are inside code blocks
                        # Skip if it starts with a language identifier
                        lines = part.strip().split('\n')
                        if lines and not lines[0].strip().isalpha():
                            return part.strip()
                        elif len(lines) > 1:
                            # Try without first line
                            return '\n'.join(lines[1:]).strip()
            except:
                pass

        # Strategy 3: Look for YAML-like patterns
        lines = response.split('\n')
        yaml_lines = []
        in_yaml = False

        for line in lines:
            stripped = line.strip()

            # Detect start of YAML-like content
            if ':' in stripped and not stripped.startswith('#'):
                in_yaml = True
                yaml_lines.append(line)
            elif in_yaml:
                if stripped == '' or stripped.startswith(' ') or stripped.startswith('-') or ':' in stripped:
                    yaml_lines.append(line)
                else:
                    # Potential end of YAML
                    break

        if yaml_lines:
            return '\n'.join(yaml_lines).strip()

        # Strategy 4: Return entire response if it looks like YAML
        if ':' in response and not response.strip().startswith('<'):
            return response.strip()

        return ""
    # ===== SERVER SETUP =====

    def setup_a2a_server(self, host: str = "0.0.0.0", port: int = 5000, **kwargs):
        """Setup A2A server for bidirectional communication"""
        if not A2A_AVAILABLE:
            wprint("A2A not available, cannot setup server")
            return

        try:
            self.a2a_server = A2AServer(
                host=host,
                port=port,
                agent_card=AgentCard(
                    name=self.amd.name,
                    description="Production-ready PocketFlow agent",
                    version="1.0.0"
                ),
                **kwargs
            )

            # Register agent methods
            @self.a2a_server.route("/run")
            async def handle_run(request_data):
                query = request_data.get("query", "")
                session_id = request_data.get("session_id", "a2a_session")

                response = await self.a_run(query, session_id=session_id)
                return {"response": response}

            rprint(f"A2A server setup on {host}:{port}")

        except Exception as e:
            eprint(f"Failed to setup A2A server: {e}")

    def setup_mcp_server(self, host: str = "0.0.0.0", port: int = 8000, name: str = None, **kwargs):
        """Setup MCP server"""
        if not MCP_AVAILABLE:
            wprint("MCP not available, cannot setup server")
            return

        try:
            server_name = name or f"{self.amd.name}_MCP"
            self.mcp_server = FastMCP(server_name)

            # Register agent as MCP tool
            @self.mcp_server.tool()
            async def agent_run(query: str, session_id: str = "mcp_session") -> str:
                """Execute agent with given query"""
                return await self.a_run(query, session_id=session_id)

            rprint(f"MCP server setup: {server_name}")

        except Exception as e:
            eprint(f"Failed to setup MCP server: {e}")

    # ===== LIFECYCLE MANAGEMENT =====

    async def start_servers(self):
        """Start all configured servers"""
        tasks = []

        if self.a2a_server:
            tasks.append(asyncio.create_task(self.a2a_server.start()))

        if self.mcp_server:
            tasks.append(asyncio.create_task(self.mcp_server.run()))

        if tasks:
            rprint(f"Starting {len(tasks)} servers...")
            await asyncio.gather(*tasks, return_exceptions=True)

    def clear_context(self, session_id: str = None) -> bool:
        """Clear context über UnifiedContextManager mit Session-spezifischer Unterstützung"""
        try:
            #Clear über Context Manager
            if session_id:
                # Clear specific session
                if session_id in self.context_manager.session_managers:
                    session = self.context_manager.session_managers[session_id]
                    if hasattr(session, 'history'):
                        session.history = []
                    elif isinstance(session, dict) and 'history' in session:
                        session['history'] = []

                    # Remove from session managers
                    del self.context_manager.session_managers[session_id]

                    # Clear variable manager scope for this session
                    if self.variable_manager:
                        scope_name = f'session_{session_id}'
                        if scope_name in self.variable_manager.scopes:
                            del self.variable_manager.scopes[scope_name]

                    rprint(f"Context cleared for session: {session_id}")
            else:
                # Clear all sessions
                for session_id, session in self.context_manager.session_managers.items():
                    if hasattr(session, 'history'):
                        session.history = []
                    elif isinstance(session, dict) and 'history' in session:
                        session['history'] = []

                self.context_manager.session_managers = {}
                rprint("Context cleared for all sessions")

            # Clear context cache
            self.context_manager._invalidate_cache(session_id)

            # Clear current execution context in shared
            context_keys_to_clear = [
                "current_query", "current_response", "current_plan", "tasks",
                "results", "task_plans", "session_data", "formatted_context",
                "synthesized_response", "quality_assessment", "plan_adaptations",
                "executor_performance", "llm_tool_conversation", "aggregated_context"
            ]

            for key in context_keys_to_clear:
                if key in self.shared:
                    if isinstance(self.shared[key], dict):
                        self.shared[key] = {}
                    elif isinstance(self.shared[key], list):
                        self.shared[key] = []
                    else:
                        self.shared[key] = None

            # Clear variable manager scopes (except core system variables)
            if hasattr(self, 'variable_manager'):
                # Clear user, results, tasks scopes
                self.variable_manager.register_scope('user', {})
                self.variable_manager.register_scope('results', {})
                self.variable_manager.register_scope('tasks', {})
                # Reset cache
                self.variable_manager._cache.clear()

            # Reset execution state
            self.is_running = False
            self.is_paused = False
            self.shared["system_status"] = "idle"

            # Clear progress tracking
            if hasattr(self, 'progress_tracker'):
                self.progress_tracker.reset_session_metrics()

            return True

        except Exception as e:
            eprint(f"Failed to clear context: {e}")
            return False

    async def clean_memory(self, deep_clean: bool = False) -> bool:
        """Clean memory and context of the agent"""
        try:
            # Clear current context first
            self.clear_context()

            # Clean world model
            self.shared["world_model"] = {}
            self.world_model = {}

            # Clean performance metrics
            self.shared["performance_metrics"] = {}

            # Deep clean session storage
            session_managers = self.shared.get("session_managers", {})
            if session_managers:
                for _manager_name, manager in session_managers.items():
                    if hasattr(manager, 'clear_all_history'):
                        await manager.clear_all_history()
                    elif hasattr(manager, 'clear_history'):
                        manager.clear_history()

            # Clear session managers entirely
            self.shared["session_managers"] = {}
            self.shared["session_initialized"] = False

            # Clean variable manager completely
            if hasattr(self, 'variable_manager'):
                # Reinitialize with clean state
                self.variable_manager = VariableManager({}, self.shared)
                self._setup_variable_scopes()

            # Clean tool analysis cache if deep clean
            if deep_clean:
                self._tool_capabilities = {}
                self._tool_analysis_cache = {}

                # Remove tool analysis file
                if hasattr(self, 'tool_analysis_file') and os.path.exists(self.tool_analysis_file):
                    try:
                        os.remove(self.tool_analysis_file)
                        rprint("Removed tool analysis cache file")
                    except:
                        pass

            # Clean checkpoint data
            self.checkpoint_data = {}
            self.last_checkpoint = None

            # Clean execution history
            if hasattr(self.task_flow, 'executor_node'):
                self.task_flow.executor_node.execution_history = []
                self.task_flow.executor_node.results_store = {}

            # Clean context manager sessions
            if hasattr(self.task_flow, 'context_manager'):
                self.task_flow.context_manager.session_managers = {}

            # Clean LLM call statistics
            self.shared.pop("llm_call_stats", None)

            # Force garbage collection
            import gc
            gc.collect()

            rprint(f"Memory cleaned (deep_clean: {deep_clean})")
            return True

        except Exception as e:
            eprint(f"Failed to clean memory: {e}")
            return False

    async def close(self):
        """Clean shutdown"""
        self.is_running = False
        self._shutdown_event.set()

        # Create final checkpoint
        if self.enable_pause_resume:
            checkpoint = await self._create_checkpoint()
            await self._save_checkpoint(checkpoint, "final_checkpoint.pkl")

        # Shutdown executor
        self.executor.shutdown(wait=True)

        # Close servers
        if self.a2a_server:
            await self.a2a_server.close()

        if self.mcp_server:
            await self.mcp_server.close()

        if hasattr(self, '_mcp_session_manager'):
            await self._mcp_session_manager.cleanup_all()

        rprint("Agent shutdown complete")

    @property
    def total_cost(self) -> float:
        """Get total cost if budget manager available"""
        if hasattr(self.amd, 'budget_manager') and self.amd.budget_manager:
            return getattr(self.amd.budget_manager, 'total_cost', 0.0)
        return 0.0

    async def get_context_overview(self, session_id: str = None, display: bool = False) -> dict[str, Any]:
        """
        Detaillierte Übersicht des aktuellen Contexts mit Token-Counts und optionaler Display-Darstellung

        Args:
            session_id: Session ID für context (default: active_session)
            display: Ob die Übersicht im Terminal-Style angezeigt werden soll

        Returns:
            dict: Detaillierte Context-Übersicht mit Raw-Daten und Token-Counts
        """
        try:
            session_id = session_id or self.active_session or "default"

            # Token counting function
            def count_tokens(text: str) -> int:
                """Einfache Token-Approximation (4 chars ≈ 1 token für deutsche/englische Texte)"""
                try:
                    from litellm import token_counter
                    return token_counter(self.amd.fast_llm_model, text=text)
                except:
                    pass
                return max(1, len(str(text)) // 4)

            context_overview = {
                "session_info": {
                    "session_id": session_id,
                    "agent_name": self.amd.name,
                    "timestamp": datetime.now().isoformat(),
                    "active_session": self.active_session,
                    "is_running": self.is_running
                },
                "system_prompt": {},
                "meta_tools": {},
                "agent_tools": {},
                "mcp_tools": {},
                "variables": {},
                "system_history": {},
                "unified_context": {},
                "reasoning_context": {},
                "llm_tool_context": {},
                "token_summary": {}
            }

            # === SYSTEM PROMPT ANALYSIS ===
            system_message = self.amd.get_system_message_with_persona()
            context_overview["system_prompt"] = {
                "raw_data": system_message,
                "token_count": count_tokens(system_message),
                "components": {
                    "base_message": self.amd.system_message,
                    "persona_active": self.amd.persona is not None,
                    "persona_name": self.amd.persona.name if self.amd.persona else None,
                    "persona_integration": self.amd.persona.apply_method if self.amd.persona else None
                }
            }

            # === META TOOLS ANALYSIS ===
            if hasattr(self.task_flow, 'llm_reasoner') and hasattr(self.task_flow.llm_reasoner, 'meta_tools_registry'):
                meta_tools = self.task_flow.llm_reasoner.meta_tools_registry
            else:
                meta_tools = {}

            meta_tools_info = ""
            for tool_name, tool_info in meta_tools.items():
                tool_desc = tool_info.get("description", "No description")
                meta_tools_info += f"{tool_name}: {tool_desc}\n"

            # Standard Meta-Tools
            standard_meta_tools = [
                "internal_reasoning", "manage_internal_task_stack", "delegate_to_llm_tool_node",
                "create_and_execute_plan", "advance_outline_step", "write_to_variables",
                "read_from_variables", "direct_response"
            ]

            for meta_tool in standard_meta_tools:
                meta_tools_info += f"{meta_tool}: Built-in meta-tool for agent orchestration\n"

            context_overview["meta_tools"] = {
                "raw_data": meta_tools_info,
                "token_count": count_tokens(meta_tools_info),
                "count": len(meta_tools) + len(standard_meta_tools),
                "custom_meta_tools": list(meta_tools.keys()),
                "standard_meta_tools": standard_meta_tools
            }

            # === AGENT TOOLS ANALYSIS ===
            tools_info = ""
            tool_capabilities_text = ""

            for tool_name in self.shared.get("available_tools", []):
                tool_data = self._tool_registry.get(tool_name, {})
                description = tool_data.get("description", "No description")
                args_schema = tool_data.get("args_schema", "()")
                tools_info += f"{tool_name}{args_schema}: {description}\n"

                # Tool capabilities if available
                if tool_name in self._tool_capabilities:
                    cap = self._tool_capabilities[tool_name]
                    primary_function = cap.get("primary_function", "Unknown")
                    use_cases = cap.get("use_cases", [])
                    tool_capabilities_text += f"{tool_name}: {primary_function}\n"
                    if use_cases:
                        tool_capabilities_text += f"  Use cases: {', '.join(use_cases[:3])}\n"

            context_overview["agent_tools"] = {
                "raw_data": tools_info,
                "capabilities_data": tool_capabilities_text,
                "token_count": count_tokens(tools_info + tool_capabilities_text),
                "count": len(self.shared.get("available_tools", [])),
                "analyzed_count": len(self._tool_capabilities),
                "tool_names": self.shared.get("available_tools", []),
                "intelligence_level": "high" if self._tool_capabilities else "basic"
            }

            # === MCP TOOLS ANALYSIS ===
            # Placeholder für MCP Tools (falls implementiert)
            mcp_tools_info = "No MCP tools currently active"
            if self.mcp_server:
                mcp_tools_info = f"MCP Server active: {getattr(self.mcp_server, 'name', 'Unknown')}"

            context_overview["mcp_tools"] = {
                "raw_data": mcp_tools_info,
                "token_count": count_tokens(mcp_tools_info),
                "server_active": bool(self.mcp_server),
                "server_name": getattr(self.mcp_server, 'name', None) if self.mcp_server else None
            }

            # === VARIABLES ANALYSIS ===
            variables_text = ""
            if self.variable_manager:
                variables_text = self.variable_manager.get_llm_variable_context()
            else:
                variables_text = "No variable manager available"

            context_overview["variables"] = {
                "raw_data": variables_text,
                "token_count": count_tokens(variables_text),
                "manager_available": bool(self.variable_manager),
                "total_scopes": len(self.variable_manager.scopes) if self.variable_manager else 0,
                "scope_names": list(self.variable_manager.scopes.keys()) if self.variable_manager else []
            }

            # === SYSTEM HISTORY ANALYSIS ===
            history_text = ""
            if self.context_manager and session_id in self.context_manager.session_managers:
                session = self.context_manager.session_managers[session_id]
                if hasattr(session, 'history'):
                    history_count = len(session.history)
                    history_text = f"Session History: {history_count} messages\n"

                    # Recent messages preview
                    for msg in session.history[-3:]:
                        role = msg.get('role', 'unknown')
                        content = msg.get('content', '')[:100] + "..." if len(
                            msg.get('content', '')) > 100 else msg.get('content', '')
                        timestamp = msg.get('timestamp', '')[:19]
                        history_text += f"[{timestamp}] {role}: {content}\n"
                elif isinstance(session, dict) and 'history' in session:
                    history_count = len(session['history'])
                    history_text = f"Fallback Session History: {history_count} messages"
            else:
                history_text = "No session history available"

            context_overview["system_history"] = {
                "raw_data": history_text,
                "token_count": count_tokens(history_text),
                "session_initialized": self.shared.get("session_initialized", False),
                "context_manager_available": bool(self.context_manager),
                "session_count": len(self.context_manager.session_managers) if self.context_manager else 0
            }

            # === UNIFIED CONTEXT ANALYSIS ===
            unified_context_text = ""
            try:
                unified_context = await self.context_manager.build_unified_context(session_id, "",
                                                                                   "full") if self.context_manager else {}
                if unified_context:
                    formatted_context = self.context_manager.get_formatted_context_for_llm(unified_context)
                    unified_context_text = formatted_context
                else:
                    unified_context_text = "No unified context available"
            except Exception as e:
                unified_context_text = f"Error building unified context: {str(e)}"

            context_overview["unified_context"] = {
                "raw_data": unified_context_text,
                "token_count": count_tokens(unified_context_text),
                "build_successful": "Error" not in unified_context_text,
                "manager_available": bool(self.context_manager)
            }

            # === REASONING CONTEXT ANALYSIS ===
            reasoning_context_text = ""
            if hasattr(self.task_flow, 'llm_reasoner') and hasattr(self.task_flow.llm_reasoner, 'reasoning_context'):
                reasoning_context = self.task_flow.llm_reasoner.reasoning_context
                reasoning_context_text = f"Reasoning Context: {len(reasoning_context)} entries\n"

                # Recent reasoning entries
                for entry in reasoning_context[-3:]:
                    entry_type = entry.get('type', 'unknown')
                    content = str(entry.get('content', ''))[:150] + "..." if len(
                        str(entry.get('content', ''))) > 150 else str(entry.get('content', ''))
                    reasoning_context_text += f"  {entry_type}: {content}\n"
            else:
                reasoning_context_text = "No reasoning context available"

            context_overview["reasoning_context"] = {
                "raw_data": reasoning_context_text,
                "token_count": count_tokens(reasoning_context_text),
                "reasoner_available": hasattr(self.task_flow, 'llm_reasoner'),
                "context_entries": len(self.task_flow.llm_reasoner.reasoning_context) if hasattr(self.task_flow,
                                                                                                 'llm_reasoner') and hasattr(
                    self.task_flow.llm_reasoner, 'reasoning_context') else 0
            }

            # === LLM TOOL CONTEXT ANALYSIS ===
            llm_tool_context_text = ""
            if hasattr(self.task_flow, 'llm_tool_node'):
                llm_tool_context_text = f"LLM Tool Node available with max {self.task_flow.llm_tool_node.max_tool_calls} tool calls\n"
                if hasattr(self.task_flow.llm_tool_node, 'call_log'):
                    call_log = self.task_flow.llm_tool_node.call_log
                    llm_tool_context_text += f"Call log: {len(call_log)} entries\n"
            else:
                llm_tool_context_text = "No LLM Tool Node available"

            context_overview["llm_tool_context"] = {
                "raw_data": llm_tool_context_text,
                "token_count": count_tokens(llm_tool_context_text),
                "node_available": hasattr(self.task_flow, 'llm_tool_node'),
                "max_tool_calls": getattr(self.task_flow.llm_tool_node, 'max_tool_calls', 0) if hasattr(self.task_flow,
                                                                                                        'llm_tool_node') else 0
            }

            # === TOKEN SUMMARY ===
            total_tokens = sum([
                context_overview["system_prompt"]["token_count"],
                context_overview["meta_tools"]["token_count"],
                context_overview["agent_tools"]["token_count"],
                context_overview["mcp_tools"]["token_count"],
                context_overview["variables"]["token_count"],
                context_overview["system_history"]["token_count"],
                context_overview["unified_context"]["token_count"],
                context_overview["reasoning_context"]["token_count"],
                context_overview["llm_tool_context"]["token_count"]
            ])

            context_overview["token_summary"] = {
                "total_tokens": total_tokens,
                "breakdown": {
                    "system_prompt": context_overview["system_prompt"]["token_count"],
                    "meta_tools": context_overview["meta_tools"]["token_count"],
                    "agent_tools": context_overview["agent_tools"]["token_count"],
                    "mcp_tools": context_overview["mcp_tools"]["token_count"],
                    "variables": context_overview["variables"]["token_count"],
                    "system_history": context_overview["system_history"]["token_count"],
                    "unified_context": context_overview["unified_context"]["token_count"],
                    "reasoning_context": context_overview["reasoning_context"]["token_count"],
                    "llm_tool_context": context_overview["llm_tool_context"]["token_count"]
                },
                "percentage_breakdown": {}
            }

            # Calculate percentages
            for component, token_count in context_overview["token_summary"]["breakdown"].items():
                percentage = (token_count / total_tokens * 100) if total_tokens > 0 else 0
                context_overview["token_summary"]["percentage_breakdown"][component] = round(percentage, 1)

            # === DISPLAY OUTPUT ===
            if display:
                await self._display_context_overview(context_overview)

            return context_overview

        except Exception as e:
            eprint(f"Error generating context overview: {e}")
            return {
                "error": str(e),
                "timestamp": datetime.now().isoformat(),
                "session_id": session_id
            }

    async def _display_context_overview(self, overview: dict[str, Any]):
        """Display context overview in terminal-style format similar to the image"""
        try:
            from toolboxv2.utils.extras.Style import Spinner

            print("\n" + "=" * 80)
            print("🔍 FLOW AGENT CONTEXT OVERVIEW")
            print("=" * 80)

            # Session Info
            session_info = overview["session_info"]
            print(f"📅 Session: {session_info['session_id']} | Agent: {session_info['agent_name']}")
            print(f"⏰ Generated: {session_info['timestamp'][:19]} | Running: {session_info['is_running']}")

            # Token Summary (like in the image)
            token_summary = overview["token_summary"]
            total_tokens = token_summary["total_tokens"]
            breakdown = token_summary["percentage_breakdown"]

            print(f"\n📊 CONTEXT USAGE")
            print(f"Total Context: ~{total_tokens:,} tokens")

            # Create visual bars like in the image
            bar_length = 50

            try:mf=get_max_tokens(self.amd.fast_llm_model.split('/')[-1]);self.amd.max_tokens = mf
            except:mf = self.amd.max_tokens
            try:mc=get_max_tokens(self.amd.complex_llm_model.split('/')[-1]);self.amd.max_tokens = mf
            except:mc = self.amd.max_tokens
            components = [
                ("System prompt", breakdown.get("system_prompt", 0), "🔧"),
                ("Agent tools", breakdown.get("agent_tools", 0), "🛠️"),
                ("Meta tools", breakdown.get("meta_tools", 0), "⚡"),
                ("Variables", breakdown.get("variables", 0), "📝"),
                ("History", breakdown.get("system_history", 0), "📚"),
                ("Unified ctx", breakdown.get("unified_context", 0), "🔗"),
                ("Reasoning", breakdown.get("reasoning_context", 0), "🧠"),
                ("LLM Tools", breakdown.get("llm_tool_context", 0), "🤖"),
                ("Free Space F", mf, "⬜"),
                ("Free Space C", mc, "⬜"),

            ]

            for name, percentage, icon in components:
                if percentage > 0:
                    filled_length = int(percentage * bar_length / 100)
                    bar = "█" * filled_length + "░" * (bar_length - filled_length)
                    tokens = int(total_tokens * percentage / 100)
                    print(f"{icon} {name:13}: {bar} {percentage:5.1f}% ({tokens:,} tokens)") if not name.startswith("Free") else print(f"{icon} {name:13}: ({tokens:,} tokens) used {total_tokens/tokens*100:.3f}%")

            # Detailed breakdowns
            sections = [
                ("🔧 SYSTEM PROMPT", "system_prompt"),
                ("⚡ META TOOLS", "meta_tools"),
                ("🛠️ AGENT TOOLS", "agent_tools"),
                ("📝 VARIABLES", "variables"),
                ("📚 SYSTEM HISTORY", "system_history"),
                ("🔗 UNIFIED CONTEXT", "unified_context"),
                ("🧠 REASONING CONTEXT", "reasoning_context"),
                ("🤖 LLM TOOL CONTEXT", "llm_tool_context")
            ]

            for title, key in sections:
                section_data = overview.get(key, {})
                token_count = section_data.get("token_count", 0)

                if token_count > 0:
                    print(f"\n{title} ({token_count:,} tokens)")
                    print("-" * 50)

                    # Show component-specific info
                    if key == "agent_tools":
                        print(f"  Available tools: {section_data.get('count', 0)}")
                        print(f"  Analyzed tools: {section_data.get('analyzed_count', 0)}")
                        print(f"  Intelligence: {section_data.get('intelligence_level', 'unknown')}")
                    elif key == "variables":
                        print(f"  Manager available: {section_data.get('manager_available', False)}")
                        print(f"  Total scopes: {section_data.get('total_scopes', 0)}")
                        print(f"  Scope names: {', '.join(section_data.get('scope_names', []))}")
                    elif key == "system_history":
                        print(f"  Session initialized: {section_data.get('session_initialized', False)}")
                        print(f"  Total sessions: {section_data.get('session_count', 0)}")
                    elif key == "reasoning_context":
                        print(f"  Reasoner available: {section_data.get('reasoner_available', False)}")
                        print(f"  Context entries: {section_data.get('context_entries', 0)}")
                    elif key == "meta_tools":
                        print(f"  Total meta tools: {section_data.get('count', 0)}")
                        custom = section_data.get('custom_meta_tools', [])
                        if custom:
                            print(f"  Custom tools: {', '.join(custom)}")

                    # Show raw data preview if reasonable size
                    raw_data = section_data.get('raw_data', '')
                    if len(raw_data) <= 200:
                        print(f"  Preview: {raw_data[:200]}...")

            print("\n" + "=" * 80)
            print(f"💾 Total Context Size: ~{total_tokens:,} tokens")
            print("=" * 80 + "\n")

        except Exception as e:
            eprint(f"Error displaying context overview: {e}")
            # Fallback to simple display
            print(f"\n=== CONTEXT OVERVIEW (Fallback) ===")
            print(f"Total Tokens: {overview.get('token_summary', {}).get('total_tokens', 0):,}")
            for key, data in overview.items():
                if isinstance(data, dict) and 'token_count' in data:
                    print(f"{key}: {data['token_count']:,} tokens")
            print("=" * 40)

    async def status(self, pretty_print: bool = False) -> dict[str, Any] | str:
        """Get comprehensive agent status with optional pretty printing"""

        # Core status information
        base_status = {
            "agent_info": {
                "name": self.amd.name,
                "version": "2.0",
                "type": "FlowAgent"
            },
            "runtime_status": {
                "status": self.shared.get("system_status", "idle"),
                "is_running": self.is_running,
                "is_paused": self.is_paused,
                "uptime_seconds": (datetime.now() - getattr(self, '_start_time', datetime.now())).total_seconds()
            },
            "task_execution": {
                "total_tasks": len(self.shared.get("tasks", {})),
                "active_tasks": len([t for t in self.shared.get("tasks", {}).values() if t.status == "running"]),
                "completed_tasks": len([t for t in self.shared.get("tasks", {}).values() if t.status == "completed"]),
                "failed_tasks": len([t for t in self.shared.get("tasks", {}).values() if t.status == "failed"]),
                "plan_adaptations": self.shared.get("plan_adaptations", 0)
            },
            "conversation": {
                "turns": len(self.shared.get("conversation_history", [])),
                "session_id": self.shared.get("session_id", self.active_session),
                "current_user": self.shared.get("user_id"),
                "last_query": self.shared.get("current_query", "")[:100] + "..." if len(
                    self.shared.get("current_query", "")) > 100 else self.shared.get("current_query", "")
            },
            "capabilities": {
                "available_tools": len(self.shared.get("available_tools", [])),
                "tool_names": list(self.shared.get("available_tools", [])),
                "analyzed_tools": len(self._tool_capabilities),
                "world_model_size": len(self.shared.get("world_model", {})),
                "intelligence_level": "high" if self._tool_capabilities else "basic"
            },
            "memory_context": {
                "session_initialized": self.shared.get("session_initialized", False),
                "session_managers": len(self.shared.get("session_managers", {})),
                "context_system": "advanced_session_aware" if self.shared.get("session_initialized") else "basic",
                "variable_scopes": len(self.variable_manager.get_scope_info()) if hasattr(self,
                                                                                          'variable_manager') else 0
            },
            "performance": {
                "total_cost": self.total_cost,
                "checkpoint_enabled": self.enable_pause_resume,
                "last_checkpoint": self.last_checkpoint.isoformat() if self.last_checkpoint else None,
                "max_parallel_tasks": self.max_parallel_tasks
            },
            "servers": {
                "a2a_server": self.a2a_server is not None,
                "mcp_server": self.mcp_server is not None,
                "server_count": sum([self.a2a_server is not None, self.mcp_server is not None])
            },
            "configuration": {
                "fast_llm_model": self.amd.fast_llm_model,
                "complex_llm_model": self.amd.complex_llm_model,
                "use_fast_response": getattr(self.amd, 'use_fast_response', False),
                "max_input_tokens": getattr(self.amd, 'max_input_tokens', 8000),
                "persona_configured": self.amd.persona is not None,
                "format_config": bool(getattr(self.amd.persona, 'format_config', None)) if self.amd.persona else False
            }
        }

        # Add detailed execution summary if tasks exist
        tasks = self.shared.get("tasks", {})
        if tasks:
            task_types_used = {}
            tools_used = []
            execution_timeline = []

            for task_id, task in tasks.items():
                # Count task types
                task_type = getattr(task, 'type', 'unknown')
                task_types_used[task_type] = task_types_used.get(task_type, 0) + 1

                # Collect tools used
                if hasattr(task, 'tool_name') and task.tool_name:
                    tools_used.append(task.tool_name)

                # Timeline info
                if hasattr(task, 'started_at') and task.started_at:
                    timeline_entry = {
                        "task_id": task_id,
                        "type": task_type,
                        "started": task.started_at.isoformat(),
                        "status": getattr(task, 'status', 'unknown')
                    }
                    if hasattr(task, 'completed_at') and task.completed_at:
                        timeline_entry["completed"] = task.completed_at.isoformat()
                        timeline_entry["duration"] = (task.completed_at - task.started_at).total_seconds()
                    execution_timeline.append(timeline_entry)

            base_status["task_execution"].update({
                "task_types_used": task_types_used,
                "tools_used": list(set(tools_used)),
                "execution_timeline": execution_timeline[-5:]  # Last 5 tasks
            })

        # Add context statistics
        if hasattr(self.task_flow, 'context_manager'):
            context_manager = self.task_flow.context_manager
            base_status["memory_context"].update({
                "compression_threshold": context_manager.compression_threshold,
                "max_tokens": context_manager.max_tokens,
                "active_context_sessions": len(getattr(context_manager, 'session_managers', {}))
            })

        # Add variable system info
        if hasattr(self, 'variable_manager'):
            available_vars = self.variable_manager.get_available_variables()
            scope_info = self.variable_manager.get_scope_info()

            base_status["variable_system"] = {
                "total_scopes": len(scope_info),
                "scope_names": list(scope_info.keys()),
                "total_variables": sum(len(vars) for vars in available_vars.values()),
                "scope_details": {
                    scope: {"type": info["type"], "variables": len(available_vars.get(scope, {}))}
                    for scope, info in scope_info.items()
                }
            }

        # Add format quality info if available
        quality_assessment = self.shared.get("quality_assessment", {})
        if quality_assessment:
            quality_details = quality_assessment.get("quality_details", {})
            base_status["format_quality"] = {
                "overall_score": quality_details.get("total_score", 0.0),
                "format_adherence": quality_details.get("format_adherence", 0.0),
                "length_adherence": quality_details.get("length_adherence", 0.0),
                "content_quality": quality_details.get("base_quality", 0.0),
                "assessment": quality_assessment.get("quality_assessment", "unknown"),
                "has_suggestions": bool(quality_assessment.get("suggestions", []))
            }

        # Add LLM usage statistics
        llm_stats = self.shared.get("llm_call_stats", {})
        if llm_stats:
            base_status["llm_usage"] = {
                "total_calls": llm_stats.get("total_calls", 0),
                "context_compression_rate": llm_stats.get("context_compression_rate", 0.0),
                "average_context_tokens": llm_stats.get("context_tokens_used", 0) / max(llm_stats.get("total_calls", 1),
                                                                                        1),
                "total_tokens_used": llm_stats.get("total_tokens_used", 0)
            }

        # Add timestamp
        base_status["timestamp"] = datetime.now().isoformat()

        base_status["context_statistic"] = self.get_context_statistics()
        if not pretty_print:
            base_status["agent_context"] = await self.get_context_overview()
            return base_status

        # Pretty print using EnhancedVerboseOutput
        try:
            from toolboxv2.mods.isaa.extras.verbose_output import EnhancedVerboseOutput
            verbose_output = EnhancedVerboseOutput(verbose=True)

            # Header
            verbose_output.log_header(f"Agent Status: {base_status['agent_info']['name']}")

            # Runtime Status
            status_color = {
                "running": "SUCCESS",
                "paused": "WARNING",
                "idle": "INFO",
                "error": "ERROR"
            }.get(base_status["runtime_status"]["status"], "INFO")

            getattr(verbose_output, f"print_{status_color.lower()}")(
                f"Status: {base_status['runtime_status']['status'].upper()}"
            )

            # Task Execution Summary
            task_exec = base_status["task_execution"]
            if task_exec["total_tasks"] > 0:
                verbose_output.formatter.print_section(
                    "Task Execution",
                    f"Total: {task_exec['total_tasks']} | "
                    f"Completed: {task_exec['completed_tasks']} | "
                    f"Failed: {task_exec['failed_tasks']} | "
                    f"Active: {task_exec['active_tasks']}\n"
                    f"Adaptations: {task_exec['plan_adaptations']}"
                )

                if task_exec.get("tools_used"):
                    verbose_output.formatter.print_section(
                        "Tools Used",
                        ", ".join(task_exec["tools_used"])
                    )

            # Capabilities
            caps = base_status["capabilities"]
            verbose_output.formatter.print_section(
                "Capabilities",
                f"Intelligence Level: {caps['intelligence_level']}\n"
                f"Available Tools: {caps['available_tools']}\n"
                f"Analyzed Tools: {caps['analyzed_tools']}\n"
                f"World Model Size: {caps['world_model_size']}"
            )

            # Memory & Context
            memory = base_status["memory_context"]
            verbose_output.formatter.print_section(
                "Memory & Context",
                f"Context System: {memory['context_system']}\n"
                f"Session Managers: {memory['session_managers']}\n"
                f"Variable Scopes: {memory['variable_scopes']}\n"
                f"Session Initialized: {memory['session_initialized']}"
            )

            # Context Statistics
            stats = base_status["context_statistic"]
            verbose_output.formatter.print_section(
                "Context & Stats",
                f"Compression Stats: {stats['compression_stats']}\n"
                f"Session Usage: {stats['context_usage']}\n"
                f"Session Managers: {stats['session_managers']}\n"
            )

            # Configuration
            config = base_status["configuration"]
            verbose_output.formatter.print_section(
                "Configuration",
                f"Fast LLM: {config['fast_llm_model']}\n"
                f"Complex LLM: {config['complex_llm_model']}\n"
                f"Max Tokens: {config['max_input_tokens']}\n"
                f"Persona: {'Configured' if config['persona_configured'] else 'Default'}\n"
                f"Format Config: {'Active' if config['format_config'] else 'None'}"
            )

            # Performance
            perf = base_status["performance"]
            verbose_output.formatter.print_section(
                "Performance",
                f"Total Cost: ${perf['total_cost']:.4f}\n"
                f"Checkpointing: {'Enabled' if perf['checkpoint_enabled'] else 'Disabled'}\n"
                f"Max Parallel Tasks: {perf['max_parallel_tasks']}\n"
                f"Last Checkpoint: {perf['last_checkpoint'] or 'None'}"
            )

            # Variable System Details
            if "variable_system" in base_status:
                var_sys = base_status["variable_system"]
                scope_details = []
                for scope, details in var_sys["scope_details"].items():
                    scope_details.append(f"{scope}: {details['variables']} variables ({details['type']})")

                verbose_output.formatter.print_section(
                    "Variable System",
                    f"Total Scopes: {var_sys['total_scopes']}\n"
                    f"Total Variables: {var_sys['total_variables']}\n" +
                    "\n".join(scope_details)
                )

            # Format Quality
            if "format_quality" in base_status:
                quality = base_status["format_quality"]
                verbose_output.formatter.print_section(
                    "Format Quality",
                    f"Overall Score: {quality['overall_score']:.2f}\n"
                    f"Format Adherence: {quality['format_adherence']:.2f}\n"
                    f"Length Adherence: {quality['length_adherence']:.2f}\n"
                    f"Content Quality: {quality['content_quality']:.2f}\n"
                    f"Assessment: {quality['assessment']}"
                )

            # LLM Usage
            if "llm_usage" in base_status:
                llm = base_status["llm_usage"]
                verbose_output.formatter.print_section(
                    "LLM Usage Statistics",
                    f"Total Calls: {llm['total_calls']}\n"
                    f"Avg Context Tokens: {llm['average_context_tokens']:.1f}\n"
                    f"Total Tokens: {llm['total_tokens_used']}\n"
                    f"Compression Rate: {llm['context_compression_rate']:.2%}"
                )

            # Servers
            servers = base_status["servers"]
            if servers["server_count"] > 0:
                server_status = []
                if servers["a2a_server"]:
                    server_status.append("A2A Server: Active")
                if servers["mcp_server"]:
                    server_status.append("MCP Server: Active")

                verbose_output.formatter.print_section(
                    "Servers",
                    "\n".join(server_status)
                )

            verbose_output.print_separator()
            await self.get_context_overview(display=True)
            verbose_output.print_separator()
            verbose_output.print_info(f"Status generated at: {base_status['timestamp']}")

            return "Status printed above"

        except Exception:
            # Fallback to JSON if pretty print fails
            import json
            return json.dumps(base_status, indent=2, default=str)

    @property
    def tool_registry(self):
        return self._tool_registry

    def __rshift__(self, other):
        return Chain(self) >> other

    def __add__(self, other):
        return Chain(self) + other

    def __and__(self, other):
        return Chain(self) & other

    def __mod__(self, other):
        """Implements % operator for conditional branching"""
        return ConditionalChain(self, other)

    def bind(self, *agents, shared_scopes: list[str] = None, auto_sync: bool = True):
        """
        Bind two or more agents together with shared and private variable spaces.

        Args:
            *agents: FlowAgent instances to bind together
            shared_scopes: List of scope names to share (default: ['world', 'results', 'system'])
            auto_sync: Whether to automatically sync variables and context

        Returns:
            dict: Binding configuration with agent references
        """
        if shared_scopes is None:
            shared_scopes = ['world', 'results', 'system']

        # Create unique binding ID
        binding_id = f"bind_{int(time.time())}_{len(agents)}"

        # All agents in this binding (including self)
        all_agents = [self] + list(agents)

        # Create shared variable manager that all agents will reference
        shared_world_model = {}
        shared_state = {}

        # Merge existing data from all agents
        for agent in all_agents:
            # Merge world models
            shared_world_model.update(agent.world_model)
            shared_state.update(agent.shared)

        # Create shared variable manager
        shared_variable_manager = VariableManager(shared_world_model, shared_state)

        # Set up shared scopes with merged data
        for scope_name in shared_scopes:
            merged_scope = {}
            for agent in all_agents:
                if hasattr(agent, 'variable_manager') and agent.variable_manager:
                    agent_scope_data = agent.variable_manager.scopes.get(scope_name, {})
                    if isinstance(agent_scope_data, dict):
                        merged_scope.update(agent_scope_data)
            shared_variable_manager.register_scope(scope_name, merged_scope)

        # Create binding configuration
        binding_config = {
            'binding_id': binding_id,
            'agents': all_agents,
            'shared_scopes': shared_scopes,
            'auto_sync': auto_sync,
            'shared_variable_manager': shared_variable_manager,
            'private_managers': {},
            'created_at': datetime.now().isoformat()
        }

        # Configure each agent
        for i, agent in enumerate(all_agents):
            agent_private_id = f"{binding_id}_agent_{i}_{agent.amd.name}"

            # Create private variable manager for agent-specific data
            private_world_model = agent.world_model.copy()
            private_shared = agent.shared.copy()
            private_variable_manager = VariableManager(private_world_model, private_shared)

            # Set up private scopes (user, session-specific data, agent-specific configs)
            private_scopes = ['user', 'agent', 'session_private', 'tasks_private']
            for scope_name in private_scopes:
                if hasattr(agent, 'variable_manager') and agent.variable_manager:
                    agent_scope_data = agent.variable_manager.scopes.get(scope_name, {})
                    private_variable_manager.register_scope(f"{scope_name}_{agent.amd.name}", agent_scope_data)

            binding_config['private_managers'][agent.amd.name] = private_variable_manager

            # Replace agent's variable manager with a unified one
            unified_manager = UnifiedBindingManager(
                shared_manager=shared_variable_manager,
                private_manager=private_variable_manager,
                agent_name=agent.amd.name,
                shared_scopes=shared_scopes,
                auto_sync=auto_sync,
                binding_config=binding_config
            )

            # Store original managers for unbinding
            if not hasattr(agent, '_original_managers'):
                agent._original_managers = {
                    'variable_manager': agent.variable_manager,
                    'world_model': agent.world_model.copy(),
                    'shared': agent.shared.copy()
                }

            # Set new unified manager
            agent.variable_manager = unified_manager
            agent.world_model = shared_world_model
            agent.shared = shared_state

            # Update shared state with binding info
            agent.shared['binding_config'] = binding_config
            agent.shared['is_bound'] = True
            agent.shared['binding_id'] = binding_id
            agent.shared['bound_agents'] = [a.amd.name for a in all_agents]

            # Sync context manager if available
            if hasattr(agent, 'context_manager') and agent.context_manager:
                agent.context_manager.variable_manager = unified_manager

                # Share session managers between bound agents if auto_sync is enabled
                if auto_sync:
                    # Merge session managers from all agents
                    all_sessions = {}
                    for bound_agent in all_agents:
                        if hasattr(bound_agent, 'context_manager') and bound_agent.context_manager:
                            if hasattr(bound_agent.context_manager, 'session_managers'):
                                all_sessions.update(bound_agent.context_manager.session_managers)

                    # Update all agents with merged sessions
                    for bound_agent in all_agents:
                        if hasattr(bound_agent, 'context_manager') and bound_agent.context_manager:
                            bound_agent.context_manager.session_managers.update(all_sessions)

        # Set up auto-sync mechanism if enabled
        if auto_sync:
            binding_config['sync_handler'] = BindingSyncHandler(binding_config)

        rprint(f"Successfully bound {len(all_agents)} agents together (Binding ID: {binding_id})")
        rprint(f"Shared scopes: {', '.join(shared_scopes)}")
        rprint(f"Bound agents: {', '.join([agent.amd.name for agent in all_agents])}")

        return binding_config

    def unbind(self, preserve_shared_data: bool = False):
        """
        Unbind this agent from any binding configuration.

        Args:
            preserve_shared_data: Whether to preserve shared data in the agent after unbinding

        Returns:
            dict: Unbinding result with statistics
        """
        if not self.shared.get('is_bound', False):
            return {
                'success': False,
                'message': f"Agent {self.amd.name} is not currently bound to any other agents"
            }

        binding_config = self.shared.get('binding_config')
        if not binding_config:
            return {
                'success': False,
                'message': "No binding configuration found"
            }

        binding_id = binding_config['binding_id']
        bound_agents = binding_config['agents']

        unbind_stats = {
            'binding_id': binding_id,
            'agents_affected': [],
            'shared_data_preserved': preserve_shared_data,
            'private_data_restored': False,
            'unbind_timestamp': datetime.now().isoformat()
        }

        try:
            # Restore original managers for this agent
            if hasattr(self, '_original_managers'):
                original = self._original_managers

                if preserve_shared_data:
                    # Merge current shared data with original data
                    if isinstance(original['world_model'], dict):
                        original['world_model'].update(self.world_model)
                    if isinstance(original['shared'], dict):
                        original['shared'].update({k: v for k, v in self.shared.items()
                                                   if k not in ['binding_config', 'is_bound', 'binding_id',
                                                                'bound_agents']})

                # Restore original variable manager
                self.variable_manager = original['variable_manager']
                self.world_model = original['world_model']
                self.shared = original['shared']

                # Update context manager
                if hasattr(self, 'context_manager') and self.context_manager:
                    self.context_manager.variable_manager = self.variable_manager

                unbind_stats['private_data_restored'] = True
                del self._original_managers

            # Clean up binding state
            self.shared.pop('binding_config', None)
            self.shared.pop('is_bound', None)
            self.shared.pop('binding_id', None)
            self.shared.pop('bound_agents', None)

            # Update binding configuration to remove this agent
            remaining_agents = [agent for agent in bound_agents if agent != self]
            if remaining_agents:
                # Update binding config for remaining agents
                binding_config['agents'] = remaining_agents
                for agent in remaining_agents:
                    if hasattr(agent, 'shared') and agent.shared.get('is_bound'):
                        agent.shared['bound_agents'] = [a.amd.name for a in remaining_agents]

            unbind_stats['agents_affected'] = [agent.amd.name for agent in bound_agents]

            # Clean up sync handler if this was the last agent
            if len(remaining_agents) <= 1:
                sync_handler = binding_config.get('sync_handler')
                if sync_handler and hasattr(sync_handler, 'cleanup'):
                    sync_handler.cleanup()

            rprint(f"Agent {self.amd.name} successfully unbound from binding {binding_id}")
            rprint(f"Shared data preserved: {preserve_shared_data}")

            return {
                'success': True,
                'stats': unbind_stats,
                'message': f"Agent {self.amd.name} unbound successfully"
            }

        except Exception as e:
            eprint(f"Error during unbinding: {e}")
            return {
                'success': False,
                'error': str(e),
                'stats': unbind_stats
            }

total_cost property

Get total cost if budget manager available

__mod__(other)

Implements % operator for conditional branching

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

def __mod__(self, other):
    """Implements % operator for conditional branching"""
    return ConditionalChain(self, other)

a_format_class(pydantic_model, prompt, message_context=None, max_retries=2, auto_context=True, session_id=None, **kwargs) async

State-of-the-art LLM-based structured data formatting using Pydantic models.

Parameters:

Name	Type	Description	Default
pydantic_model	type[BaseModel]	The Pydantic model class to structure the response	required
prompt	str	The main prompt for the LLM	required
message_context	list[dict]	Optional conversation context messages	None
max_retries	int	Maximum number of retry attempts	2

Returns:

Name	Type	Description
dict	dict[str, Any]	Validated structured data matching the Pydantic model

Raises:

Type	Description
ValidationError	If the LLM response cannot be validated against the model
RuntimeError	If all retry attempts fail

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

    async def a_format_class(self,
                             pydantic_model: type[BaseModel],
                             prompt: str,
                             message_context: list[dict] = None,
                             max_retries: int = 2, auto_context=True, session_id: str = None, **kwargs) -> dict[str, Any]:
        """
        State-of-the-art LLM-based structured data formatting using Pydantic models.

        Args:
            pydantic_model: The Pydantic model class to structure the response
            prompt: The main prompt for the LLM
            message_context: Optional conversation context messages
            max_retries: Maximum number of retry attempts

        Returns:
            dict: Validated structured data matching the Pydantic model

        Raises:
            ValidationError: If the LLM response cannot be validated against the model
            RuntimeError: If all retry attempts fail
        """

        if not LITELLM_AVAILABLE:
            raise RuntimeError("LiteLLM is required for structured formatting but not available")

        if session_id and self.active_session != session_id:
            self.active_session = session_id
        # Generate schema documentation
        schema = pydantic_model.model_json_schema() if issubclass(pydantic_model, BaseModel) else (json.loads(pydantic_model) if isinstance(pydantic_model, str) else pydantic_model)
        model_name = pydantic_model.__name__ if hasattr(pydantic_model, "__name__") else (pydantic_model.get("title", "UnknownModel") if isinstance(pydantic_model, dict) else "UnknownModel")

        # Create enhanced prompt with schema
        enhanced_prompt = f"""
    {prompt}

    CRITICAL FORMATTING REQUIREMENTS:
    1. Respond ONLY in valid YAML format
    2. Follow the exact schema structure provided
    3. Use appropriate data types (strings, lists, numbers, booleans)
    4. Include ALL required fields
    5. No additional comments, explanations, or text outside the YAML

    SCHEMA FOR {model_name}:
    {yaml.dump(schema, default_flow_style=False, indent=2)}

    EXAMPLE OUTPUT FORMAT:
    ```yaml
    # Your response here following the schema exactly
    field_name: "value"
    list_field:
      - "item1"
      - "item2"
    boolean_field: true
    number_field: 42
Respond in YAML format only:
"""
        # Prepare messages
        messages = []
        if message_context:
            messages.extend(message_context)
        messages.append({"role": "user", "content": enhanced_prompt})

        # Retry logic with progressive adjustments
        last_error = None

        for attempt in range(max_retries + 1):
            try:
                # Adjust parameters based on attempt
                temperature = 0.1 + (attempt * 0.1)  # Increase temperature slightly on retries
                max_tokens = min(2000 + (attempt * 500), 4000)  # Increase token limit on retries

                rprint(f"[{model_name}] Attempt {attempt + 1}/{max_retries + 1} (temp: {temperature})")

                # Generate LLM response
                response = await self.a_run_llm_completion(
                    model=self.amd.complex_llm_model,
                    messages=messages,
                    stream=False,
                    with_context=auto_context,
                    temperature=temperature,
                    max_tokens=max_tokens,
                    task_id=f"format_{model_name.lower()}_{attempt}"
                )

                if not response or not response.strip():
                    raise ValueError("Empty response from LLM")

                # Extract YAML content with multiple fallback strategies

                yaml_content = self._extract_yaml_content(response)


                if not yaml_content:
                    raise ValueError("No valid YAML content found in response")

                # Parse YAML
                try:
                    parsed_data = yaml.safe_load(yaml_content)
                except yaml.YAMLError as e:
                    raise ValueError(f"Invalid YAML syntax: {e}")
                iprint(parsed_data)
                if not isinstance(parsed_data, dict):
                    raise ValueError(f"Expected dict, got {type(parsed_data)}")

                # Validate against Pydantic model
                try:
                    if isinstance(pydantic_model, BaseModel):
                        validated_instance = pydantic_model.model_validate(parsed_data)
                        validated_data = validated_instance.model_dump()
                    else:
                        validated_data = parsed_data

                    rprint(f"✅ Successfully formatted {model_name} on attempt {attempt + 1}")
                    return validated_data

                except ValidationError as e:
                    detailed_errors = []
                    for error in e.errors():
                        field_path = " -> ".join(str(x) for x in error['loc'])
                        detailed_errors.append(f"Field '{field_path}': {error['msg']}")

                    error_msg = "Validation failed:\n" + "\n".join(detailed_errors)
                    raise ValueError(error_msg)

            except Exception as e:
                last_error = e
                wprint(f"[{model_name}] Attempt {attempt + 1} failed: {str(e)}")

                if attempt < max_retries:
                    # Add error feedback for next attempt
                    error_feedback = f"\n\nPREVIOUS ATTEMPT FAILED: {str(e)}\nPlease correct the issues and provide valid YAML matching the schema exactly."
                    messages[-1]["content"] = enhanced_prompt + error_feedback

                    # Brief delay before retry
                    # await asyncio.sleep(0.5 * (attempt + 1))
                else:
                    eprint(f"[{model_name}] All {max_retries + 1} attempts failed")

        # All attempts failed
        raise RuntimeError(f"Failed to format {model_name} after {max_retries + 1} attempts. Last error: {last_error}")

a_run(query, session_id='default', user_id=None, stream_callback=None, remember=True, **kwargs) async

Main entry point für Agent-Ausführung mit UnifiedContextManager

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

async def a_run(
    self,
    query: str,
    session_id: str = "default",
    user_id: str = None,
    stream_callback: Callable = None,
    remember: bool = True,
    **kwargs
) -> str:
    """Main entry point für Agent-Ausführung mit UnifiedContextManager"""

    execution_start = self.progress_tracker.start_timer("total_execution")
    self.active_session = session_id
    result = None
    await self.progress_tracker.emit_event(ProgressEvent(
        event_type="execution_start",
        timestamp=time.time(),
        status=NodeStatus.RUNNING,
        node_name="FlowAgent",
        session_id=session_id,
        metadata={"query": query, "user_id": user_id}
    ))

    try:
        #Initialize or get session über UnifiedContextManager
        await self.initialize_session_context(session_id, max_history=200)

        #Store user message immediately in ChatSession wenn remember=True
        if remember:
            await self.context_manager.add_interaction(
                session_id,
                'user',
                query,
                metadata={"user_id": user_id}
            )

        # Set user context variables
        timestamp = datetime.now()
        self.variable_manager.register_scope('user', {
            'id': user_id,
            'session': session_id,
            'query': query,
            'timestamp': timestamp.isoformat()
        })

        # Update system variables
        self.variable_manager.set('system_context.timestamp', {'isoformat': timestamp.isoformat()})
        self.variable_manager.set('system_context.current_session', session_id)
        self.variable_manager.set('system_context.current_user', user_id)
        self.variable_manager.set('system_context.last_query', query)

        # Initialize with tool awareness
        await self.initialize_context_awareness()

        # VEREINFACHT: Prepare execution context - weniger Daten duplizieren
        self.shared.update({
            "current_query": query,
            "session_id": session_id,
            "user_id": user_id,
            "stream_callback": stream_callback,
            "remember": remember,
            # CENTRAL: Context Manager ist die primäre Context-Quelle
            "context_manager": self.context_manager,
            "variable_manager": self.variable_manager
        })

        # Set LLM models in shared context
        self.shared['fast_llm_model'] = self.amd.fast_llm_model
        self.shared['complex_llm_model'] = self.amd.complex_llm_model
        self.shared['persona_config'] = self.amd.persona
        self.shared['use_fast_response'] = self.amd.use_fast_response

        # Set system status
        self.shared["system_status"] = "running"
        self.is_running = True

        # Execute main orchestration flow
        result = await self._orchestrate_execution()

        #Store assistant response in ChatSession wenn remember=True
        if remember:
            await self.context_manager.add_interaction(
                session_id,
                'assistant',
                result,
                metadata={"user_id": user_id, "execution_duration": time.time() - execution_start}
            )

        total_duration = self.progress_tracker.end_timer("total_execution")

        await self.progress_tracker.emit_event(ProgressEvent(
            event_type="execution_complete",
            timestamp=time.time(),
            node_name="FlowAgent",
            status=NodeStatus.COMPLETED,
            node_duration=total_duration,
            session_id=session_id,
            metadata={
                "result_length": len(result),
                "summary": self.progress_tracker.get_summary(),
                "remembered": remember
            }
        ))

        # Checkpoint if needed
        if self.enable_pause_resume:
            with Spinner("Creating checkpoint..."):
                await self._maybe_checkpoint()
        return result

    except Exception as e:
        eprint(f"Agent execution failed: {e}", exc_info=True)
        error_response = f"I encountered an error: {str(e)}"
        result = error_response
        import traceback
        print(traceback.format_exc())

        # Store error in ChatSession wenn remember=True
        if remember:
            await self.context_manager.add_interaction(
                session_id,
                'assistant',
                error_response,
                metadata={
                    "user_id": user_id,
                    "error": True,
                    "error_type": type(e).__name__
                }
            )

        total_duration = self.progress_tracker.end_timer("total_execution")

        await self.progress_tracker.emit_event(ProgressEvent(
            event_type="error",
            timestamp=time.time(),
            node_name="FlowAgent",
            status=NodeStatus.FAILED,
            node_duration=total_duration,
            session_id=session_id,
            metadata={"error": str(e), "error_type": type(e).__name__}
        ))

        return error_response

    finally:
        self.shared["system_status"] = "idle"
        self.is_running = False
        self.active_session = None

a_run_with_format(query, response_format='frei-text', text_length='chat-conversation', custom_instructions='', **kwargs) async

Führe Agent mit spezifischem Format aus

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

async def a_run_with_format(
    self,
    query: str,
    response_format: str = "frei-text",
    text_length: str = "chat-conversation",
    custom_instructions: str = "",
    **kwargs
) -> str:
    """Führe Agent mit spezifischem Format aus"""

    # Temporäre Format-Einstellung
    original_persona = self.amd.persona

    try:
        self.set_response_format(response_format, text_length, custom_instructions)
        response = await self.a_run(query, **kwargs)
        return response
    finally:
        # Restore original persona
        self.amd.persona = original_persona
        self.shared["persona_config"] = original_persona

add_custom_flow(flow, name)

Add a custom flow for dynamic execution

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

def add_custom_flow(self, flow: AsyncFlow, name: str):
    """Add a custom flow for dynamic execution"""
    self.add_tool(flow.run_async, name=name, description=f"Custom flow: {flow.__class__.__name__}")
    rprint(f"Custom node added: {name}")

add_first_class_tool(tool_func, name, description)

Add a first-class meta-tool that can be used by the LLMReasonerNode. These are different from regular tools - they control agent sub-systems.

Parameters:

Name	Type	Description	Default
tool_func	Callable	The function to register as a meta-tool	required
name	str	Name of the meta-tool	required
description	str	Description of when and how to use it	required

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

def add_first_class_tool(self, tool_func: Callable, name: str, description: str):
    """
    Add a first-class meta-tool that can be used by the LLMReasonerNode.
    These are different from regular tools - they control agent sub-systems.

    Args:
        tool_func: The function to register as a meta-tool
        name: Name of the meta-tool
        description: Description of when and how to use it
    """

    if not asyncio.iscoroutinefunction(tool_func):
        @wraps(tool_func)
        async def async_wrapper(*args, **kwargs):
            return await asyncio.to_thread(tool_func, *args, **kwargs)

        effective_func = async_wrapper
    else:
        effective_func = tool_func

    tool_name = name or effective_func.__name__
    tool_description = description or effective_func.__doc__ or "No description"

    # Validate the tool function
    if not callable(tool_func):
        raise ValueError("Tool function must be callable")

    # Register in the reasoner's meta-tool registry (if reasoner exists)
    if hasattr(self.task_flow, 'llm_reasoner'):
        if not hasattr(self.task_flow.llm_reasoner, 'meta_tools_registry'):
            self.task_flow.llm_reasoner.meta_tools_registry = {}

        self.task_flow.llm_reasoner.meta_tools_registry[tool_name] = {
            "function": effective_func,
            "description": tool_description,
            "args_schema": get_args_schema(tool_func)
        }

        rprint(f"First-class meta-tool added: {tool_name}")
    else:
        wprint("LLMReasonerNode not available for first-class tool registration")

add_tool(tool_func, name=None, description=None, is_new=False) async

Enhanced tool addition with intelligent analysis

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

async def add_tool(self, tool_func: Callable, name: str = None, description: str = None, is_new=False):
    """Enhanced tool addition with intelligent analysis"""
    if not asyncio.iscoroutinefunction(tool_func):
        @wraps(tool_func)
        async def async_wrapper(*args, **kwargs):
            return await asyncio.to_thread(tool_func, *args, **kwargs)

        effective_func = async_wrapper
    else:
        effective_func = tool_func

    tool_name = name or effective_func.__name__
    tool_description = description or effective_func.__doc__ or "No description"

    # Store in registry
    self._tool_registry[tool_name] = {
        "function": effective_func,
        "description": tool_description,
        "args_schema": get_args_schema(tool_func)
    }

    # Add to available tools list
    if tool_name not in self.shared["available_tools"]:
        self.shared["available_tools"].append(tool_name)

    # Intelligent tool analysis
    if is_new:
        await self._analyze_tool_capabilities(tool_name, tool_description)

    rprint(f"Tool added with analysis: {tool_name}")

arun_function(function_name, *args, **kwargs) async

Asynchronously finds a function by its string name, executes it with the given arguments, and returns the result.

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

async def arun_function(self, function_name: str, *args, **kwargs) -> Any:
    """
    Asynchronously finds a function by its string name, executes it with
    the given arguments, and returns the result.
    """
    rprint(f"Attempting to run function: {function_name} with args: {args}, kwargs: {kwargs}")
    target_function = self.get_tool_by_name(function_name)

    start_time = time.perf_counter()
    if not target_function:
        raise ValueError(f"Function '{function_name}' not found in the {self.amd.name}'s registered tools.")

    try:
        if asyncio.iscoroutinefunction(target_function):
            result = await target_function(*args, **kwargs)
        else:
            # If the function is not async, run it in a thread pool
            loop = asyncio.get_running_loop()
            result = await loop.run_in_executor(None, lambda: target_function(*args, **kwargs))

        if asyncio.iscoroutine(result):
            result = await result

        if self.progress_tracker:
            await self.progress_tracker.emit_event(ProgressEvent(
                event_type="tool_call",  # Vereinheitlicht zu tool_call
                node_name="FlowAgent",
                status=NodeStatus.COMPLETED,
                success=True,
                duration=time.perf_counter() - start_time,
                tool_name=function_name,
                tool_args=kwargs,
                tool_result=result,
                is_meta_tool=False,  # Klarstellen, dass es kein Meta-Tool ist
                metadata={
                    "result_type": type(result).__name__,
                    "result_length": len(str(result))
                }
            ))
        rprint(f"Function {function_name} completed successfully with result: {result}")
        return result

    except Exception as e:
        eprint(f"Function {function_name} execution failed: {e}")
        raise

bind(*agents, shared_scopes=None, auto_sync=True)

Bind two or more agents together with shared and private variable spaces.

Parameters:

Name	Type	Description	Default
*agents		FlowAgent instances to bind together	()
shared_scopes	list[str]	List of scope names to share (default: ['world', 'results', 'system'])	None
auto_sync	bool	Whether to automatically sync variables and context	True

Returns:

Name	Type	Description
dict		Binding configuration with agent references

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

def bind(self, *agents, shared_scopes: list[str] = None, auto_sync: bool = True):
    """
    Bind two or more agents together with shared and private variable spaces.

    Args:
        *agents: FlowAgent instances to bind together
        shared_scopes: List of scope names to share (default: ['world', 'results', 'system'])
        auto_sync: Whether to automatically sync variables and context

    Returns:
        dict: Binding configuration with agent references
    """
    if shared_scopes is None:
        shared_scopes = ['world', 'results', 'system']

    # Create unique binding ID
    binding_id = f"bind_{int(time.time())}_{len(agents)}"

    # All agents in this binding (including self)
    all_agents = [self] + list(agents)

    # Create shared variable manager that all agents will reference
    shared_world_model = {}
    shared_state = {}

    # Merge existing data from all agents
    for agent in all_agents:
        # Merge world models
        shared_world_model.update(agent.world_model)
        shared_state.update(agent.shared)

    # Create shared variable manager
    shared_variable_manager = VariableManager(shared_world_model, shared_state)

    # Set up shared scopes with merged data
    for scope_name in shared_scopes:
        merged_scope = {}
        for agent in all_agents:
            if hasattr(agent, 'variable_manager') and agent.variable_manager:
                agent_scope_data = agent.variable_manager.scopes.get(scope_name, {})
                if isinstance(agent_scope_data, dict):
                    merged_scope.update(agent_scope_data)
        shared_variable_manager.register_scope(scope_name, merged_scope)

    # Create binding configuration
    binding_config = {
        'binding_id': binding_id,
        'agents': all_agents,
        'shared_scopes': shared_scopes,
        'auto_sync': auto_sync,
        'shared_variable_manager': shared_variable_manager,
        'private_managers': {},
        'created_at': datetime.now().isoformat()
    }

    # Configure each agent
    for i, agent in enumerate(all_agents):
        agent_private_id = f"{binding_id}_agent_{i}_{agent.amd.name}"

        # Create private variable manager for agent-specific data
        private_world_model = agent.world_model.copy()
        private_shared = agent.shared.copy()
        private_variable_manager = VariableManager(private_world_model, private_shared)

        # Set up private scopes (user, session-specific data, agent-specific configs)
        private_scopes = ['user', 'agent', 'session_private', 'tasks_private']
        for scope_name in private_scopes:
            if hasattr(agent, 'variable_manager') and agent.variable_manager:
                agent_scope_data = agent.variable_manager.scopes.get(scope_name, {})
                private_variable_manager.register_scope(f"{scope_name}_{agent.amd.name}", agent_scope_data)

        binding_config['private_managers'][agent.amd.name] = private_variable_manager

        # Replace agent's variable manager with a unified one
        unified_manager = UnifiedBindingManager(
            shared_manager=shared_variable_manager,
            private_manager=private_variable_manager,
            agent_name=agent.amd.name,
            shared_scopes=shared_scopes,
            auto_sync=auto_sync,
            binding_config=binding_config
        )

        # Store original managers for unbinding
        if not hasattr(agent, '_original_managers'):
            agent._original_managers = {
                'variable_manager': agent.variable_manager,
                'world_model': agent.world_model.copy(),
                'shared': agent.shared.copy()
            }

        # Set new unified manager
        agent.variable_manager = unified_manager
        agent.world_model = shared_world_model
        agent.shared = shared_state

        # Update shared state with binding info
        agent.shared['binding_config'] = binding_config
        agent.shared['is_bound'] = True
        agent.shared['binding_id'] = binding_id
        agent.shared['bound_agents'] = [a.amd.name for a in all_agents]

        # Sync context manager if available
        if hasattr(agent, 'context_manager') and agent.context_manager:
            agent.context_manager.variable_manager = unified_manager

            # Share session managers between bound agents if auto_sync is enabled
            if auto_sync:
                # Merge session managers from all agents
                all_sessions = {}
                for bound_agent in all_agents:
                    if hasattr(bound_agent, 'context_manager') and bound_agent.context_manager:
                        if hasattr(bound_agent.context_manager, 'session_managers'):
                            all_sessions.update(bound_agent.context_manager.session_managers)

                # Update all agents with merged sessions
                for bound_agent in all_agents:
                    if hasattr(bound_agent, 'context_manager') and bound_agent.context_manager:
                        bound_agent.context_manager.session_managers.update(all_sessions)

    # Set up auto-sync mechanism if enabled
    if auto_sync:
        binding_config['sync_handler'] = BindingSyncHandler(binding_config)

    rprint(f"Successfully bound {len(all_agents)} agents together (Binding ID: {binding_id})")
    rprint(f"Shared scopes: {', '.join(shared_scopes)}")
    rprint(f"Bound agents: {', '.join([agent.amd.name for agent in all_agents])}")

    return binding_config

clean_memory(deep_clean=False) async

Clean memory and context of the agent

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

async def clean_memory(self, deep_clean: bool = False) -> bool:
    """Clean memory and context of the agent"""
    try:
        # Clear current context first
        self.clear_context()

        # Clean world model
        self.shared["world_model"] = {}
        self.world_model = {}

        # Clean performance metrics
        self.shared["performance_metrics"] = {}

        # Deep clean session storage
        session_managers = self.shared.get("session_managers", {})
        if session_managers:
            for _manager_name, manager in session_managers.items():
                if hasattr(manager, 'clear_all_history'):
                    await manager.clear_all_history()
                elif hasattr(manager, 'clear_history'):
                    manager.clear_history()

        # Clear session managers entirely
        self.shared["session_managers"] = {}
        self.shared["session_initialized"] = False

        # Clean variable manager completely
        if hasattr(self, 'variable_manager'):
            # Reinitialize with clean state
            self.variable_manager = VariableManager({}, self.shared)
            self._setup_variable_scopes()

        # Clean tool analysis cache if deep clean
        if deep_clean:
            self._tool_capabilities = {}
            self._tool_analysis_cache = {}

            # Remove tool analysis file
            if hasattr(self, 'tool_analysis_file') and os.path.exists(self.tool_analysis_file):
                try:
                    os.remove(self.tool_analysis_file)
                    rprint("Removed tool analysis cache file")
                except:
                    pass

        # Clean checkpoint data
        self.checkpoint_data = {}
        self.last_checkpoint = None

        # Clean execution history
        if hasattr(self.task_flow, 'executor_node'):
            self.task_flow.executor_node.execution_history = []
            self.task_flow.executor_node.results_store = {}

        # Clean context manager sessions
        if hasattr(self.task_flow, 'context_manager'):
            self.task_flow.context_manager.session_managers = {}

        # Clean LLM call statistics
        self.shared.pop("llm_call_stats", None)

        # Force garbage collection
        import gc
        gc.collect()

        rprint(f"Memory cleaned (deep_clean: {deep_clean})")
        return True

    except Exception as e:
        eprint(f"Failed to clean memory: {e}")
        return False

cleanup_session_context(session_id=None, keep_count=100, remove_old_snapshots=True) async

Cleanup session context by removing old snapshots and entries

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

async def cleanup_session_context(self, session_id: str = None, keep_count: int = 100,
                                  remove_old_snapshots: bool = True) -> dict[str, Any]:
    """Cleanup session context by removing old snapshots and entries"""
    try:
        session_id = session_id or self.shared.get("session_id", "default")

        if not self.context_manager:
            return {"error": "Context manager not available"}

        session = self.context_manager.session_managers.get(session_id)
        if not session or not hasattr(session, 'history'):
            return {"error": f"Session {session_id} not found or has no history"}

        cleanup_stats = {
            "original_message_count": len(session.history),
            "context_snapshots_removed": 0,
            "context_entries_removed": 0,
            "regular_messages_kept": 0,
            "cleanup_performed": False
        }

        if len(session.history) <= keep_count:
            return {**cleanup_stats, "message": "No cleanup needed"}

        # Separate different types of messages
        regular_messages = []
        context_snapshots = []
        context_entries = []

        for message in session.history:
            metadata = message.get("metadata", {})

            if metadata.get("is_context_snapshot"):
                context_snapshots.append(message)
            elif metadata.get("is_context_entry"):
                context_entries.append(message)
            else:
                regular_messages.append(message)

        # Keep most recent regular messages
        messages_to_keep = regular_messages[-keep_count:]
        cleanup_stats["regular_messages_kept"] = len(messages_to_keep)

        # Keep most recent context snapshots (if not removing)
        if not remove_old_snapshots:
            recent_snapshots = context_snapshots[-5:]  # Keep last 5 snapshots
            messages_to_keep.extend(recent_snapshots)
        else:
            cleanup_stats["context_snapshots_removed"] = len(context_snapshots)

        # Keep persistent context entries
        persistent_entries = [
            entry for entry in context_entries
            if entry.get("persistent", True)
        ]
        messages_to_keep.extend(persistent_entries)
        cleanup_stats["context_entries_removed"] = len(context_entries) - len(persistent_entries)

        # Sort by timestamp and update session
        messages_to_keep.sort(key=lambda x: x.get("timestamp", ""))
        session.history = messages_to_keep

        cleanup_stats.update({
            "final_message_count": len(session.history),
            "cleanup_performed": True,
            "messages_removed": cleanup_stats["original_message_count"] - len(session.history)
        })

        rprint(f"Session cleanup completed: {cleanup_stats['messages_removed']} messages removed")
        return cleanup_stats

    except Exception as e:
        eprint(f"Failed to cleanup session context: {e}")
        return {"error": str(e)}

clear_context(session_id=None)

Clear context über UnifiedContextManager mit Session-spezifischer Unterstützung

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

def clear_context(self, session_id: str = None) -> bool:
    """Clear context über UnifiedContextManager mit Session-spezifischer Unterstützung"""
    try:
        #Clear über Context Manager
        if session_id:
            # Clear specific session
            if session_id in self.context_manager.session_managers:
                session = self.context_manager.session_managers[session_id]
                if hasattr(session, 'history'):
                    session.history = []
                elif isinstance(session, dict) and 'history' in session:
                    session['history'] = []

                # Remove from session managers
                del self.context_manager.session_managers[session_id]

                # Clear variable manager scope for this session
                if self.variable_manager:
                    scope_name = f'session_{session_id}'
                    if scope_name in self.variable_manager.scopes:
                        del self.variable_manager.scopes[scope_name]

                rprint(f"Context cleared for session: {session_id}")
        else:
            # Clear all sessions
            for session_id, session in self.context_manager.session_managers.items():
                if hasattr(session, 'history'):
                    session.history = []
                elif isinstance(session, dict) and 'history' in session:
                    session['history'] = []

            self.context_manager.session_managers = {}
            rprint("Context cleared for all sessions")

        # Clear context cache
        self.context_manager._invalidate_cache(session_id)

        # Clear current execution context in shared
        context_keys_to_clear = [
            "current_query", "current_response", "current_plan", "tasks",
            "results", "task_plans", "session_data", "formatted_context",
            "synthesized_response", "quality_assessment", "plan_adaptations",
            "executor_performance", "llm_tool_conversation", "aggregated_context"
        ]

        for key in context_keys_to_clear:
            if key in self.shared:
                if isinstance(self.shared[key], dict):
                    self.shared[key] = {}
                elif isinstance(self.shared[key], list):
                    self.shared[key] = []
                else:
                    self.shared[key] = None

        # Clear variable manager scopes (except core system variables)
        if hasattr(self, 'variable_manager'):
            # Clear user, results, tasks scopes
            self.variable_manager.register_scope('user', {})
            self.variable_manager.register_scope('results', {})
            self.variable_manager.register_scope('tasks', {})
            # Reset cache
            self.variable_manager._cache.clear()

        # Reset execution state
        self.is_running = False
        self.is_paused = False
        self.shared["system_status"] = "idle"

        # Clear progress tracking
        if hasattr(self, 'progress_tracker'):
            self.progress_tracker.reset_session_metrics()

        return True

    except Exception as e:
        eprint(f"Failed to clear context: {e}")
        return False

close() async

Clean shutdown

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

async def close(self):
    """Clean shutdown"""
    self.is_running = False
    self._shutdown_event.set()

    # Create final checkpoint
    if self.enable_pause_resume:
        checkpoint = await self._create_checkpoint()
        await self._save_checkpoint(checkpoint, "final_checkpoint.pkl")

    # Shutdown executor
    self.executor.shutdown(wait=True)

    # Close servers
    if self.a2a_server:
        await self.a2a_server.close()

    if self.mcp_server:
        await self.mcp_server.close()

    if hasattr(self, '_mcp_session_manager'):
        await self._mcp_session_manager.cleanup_all()

    rprint("Agent shutdown complete")

configure_persona_integration(apply_method='system_prompt', integration_level='light')

Configure how persona is applied

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

def configure_persona_integration(self, apply_method: str = "system_prompt", integration_level: str = "light"):
    """Configure how persona is applied"""
    if self.amd.persona:
        self.amd.persona.apply_method = apply_method
        self.amd.persona.integration_level = integration_level
        rprint(f"Persona integration updated: {apply_method}, {integration_level}")
    else:
        wprint("No persona configured to update")

delete_old_checkpoints(keep_count=5, max_age_hours=168) async

Delete old checkpoints, keeping the most recent ones

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

async def delete_old_checkpoints(self, keep_count: int = 5, max_age_hours: int = 168) -> dict[str, Any]:
    """Delete old checkpoints, keeping the most recent ones"""
    try:
        checkpoints = self.list_available_checkpoints(
            max_age_hours=max_age_hours * 2)  # Look further back for deletion

        deleted_count = 0
        deleted_size_kb = 0
        errors = []

        if len(checkpoints) > keep_count:
            # Keep the newest, delete the rest (except final checkpoint)
            to_delete = checkpoints[keep_count:]

            for checkpoint in to_delete:
                if checkpoint["checkpoint_type"] != "final":  # Never delete final checkpoint
                    try:
                        os.remove(checkpoint["filepath"])
                        deleted_count += 1
                        deleted_size_kb += checkpoint["file_size_kb"]
                        rprint(f"Deleted old checkpoint: {checkpoint['filename']}")
                    except Exception as e:
                        import traceback
                        print(traceback.format_exc())
                        errors.append(f"Failed to delete {checkpoint['filename']}: {e}")

        # Also delete checkpoints older than max_age_hours
        old_checkpoints = [cp for cp in checkpoints if
                           cp["age_hours"] > max_age_hours and cp["checkpoint_type"] != "final"]
        for checkpoint in old_checkpoints:
            if checkpoint not in checkpoints[keep_count:]:  # Don't double-delete
                try:
                    os.remove(checkpoint["filepath"])
                    deleted_count += 1
                    deleted_size_kb += checkpoint["file_size_kb"]
                    rprint(f"Deleted aged checkpoint: {checkpoint['filename']}")
                except Exception as e:
                    import traceback
                    print(traceback.format_exc())
                    errors.append(f"Failed to delete {checkpoint['filename']}: {e}")

        return {
            "success": True,
            "deleted_count": deleted_count,
            "freed_space_kb": round(deleted_size_kb, 1),
            "remaining_checkpoints": len(checkpoints) - deleted_count,
            "errors": errors
        }

    except Exception as e:
        import traceback
        print(traceback.format_exc())
        eprint(f"Failed to delete old checkpoints: {e}")
        return {
            "success": False,
            "error": str(e),
            "deleted_count": 0
        }

explain_reasoning_process() async

Erkläre den Reasoning-Prozess des Agenten

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

    async def explain_reasoning_process(self) -> str:
        """Erkläre den Reasoning-Prozess des Agenten"""
        if not LITELLM_AVAILABLE:
            return "Reasoning explanation requires LLM capabilities."

        summary = await self.get_task_execution_summary()

        prompt = f"""
Erkläre den Reasoning-Prozess dieses AI-Agenten in verständlicher Form:

## Ausführungszusammenfassung
- Total Tasks: {summary['total_tasks']}
- Erfolgreich: {len(summary['completed_tasks'])}
- Fehlgeschlagen: {len(summary['failed_tasks'])}
- Plan-Adaptationen: {summary['adaptations']}
- Verwendete Tools: {', '.join(set(summary['tools_used']))}
- Task-Typen: {summary['task_types_used']}

## Task-Details
Erfolgreiche Tasks:
{self._format_tasks_for_explanation(summary['completed_tasks'])}

## Anweisungen
Erkläre in 2-3 Absätzen:
1. Welche Strategie der Agent gewählt hat
2. Wie er die Aufgabe in Tasks unterteilt hat
3. Wie er auf unerwartete Ergebnisse reagiert hat (falls Adaptationen)
4. Was die wichtigsten Erkenntnisse waren

Schreibe für einen technischen Nutzer, aber verständlich."""

        try:
            response = await self.a_run_llm_completion(
                model=self.amd.complex_llm_model,
                messages=[{"role": "user", "content": prompt}],
                temperature=0.5,
                max_tokens=800,task_id="reasoning_explanation"
            )

            return response

        except Exception as e:
            import traceback
            print(traceback.format_exc())
            return f"Could not generate reasoning explanation: {e}"

format_text(text, **context)

Format text with variables

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

def format_text(self, text: str, **context) -> str:
    """Format text with variables"""
    return self.variable_manager.format_text(text, context)

get_available_formats()

Erhalte verfügbare Format- und Längen-Optionen

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

def get_available_formats(self) -> dict[str, list[str]]:
    """Erhalte verfügbare Format- und Längen-Optionen"""
    return {
        "formats": [f.value for f in ResponseFormat],
        "lengths": [l.value for l in TextLength],
        "format_descriptions": {
            f.value: FormatConfig(response_format=f).get_format_instructions()
            for f in ResponseFormat
        },
        "length_descriptions": {
            l.value: FormatConfig(text_length=l).get_length_instructions()
            for l in TextLength
        }
    }

get_available_variables()

Get available variables for dynamic formatting

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

def get_available_variables(self) -> dict[str, str]:
    """Get available variables for dynamic formatting"""
    return self.variable_manager.get_available_variables()

get_context(session_id=None, format_for_llm=True) async

ÜBERARBEITET: Get context über UnifiedContextManager statt verteilte Quellen

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

async def get_context(self, session_id: str = None, format_for_llm: bool = True) -> str | dict[str, Any]:
    """
    ÜBERARBEITET: Get context über UnifiedContextManager statt verteilte Quellen
    """
    try:
        session_id = session_id or self.shared.get("session_id", self.active_session)
        query = self.shared.get("current_query", "")

        #Hole unified context über Context Manager
        unified_context = await self.context_manager.build_unified_context(session_id, query, "full")


        if format_for_llm:
            return self.context_manager.get_formatted_context_for_llm(unified_context)
        else:
            return unified_context

    except Exception as e:
        import traceback
        print(traceback.format_exc())
        eprint(f"Failed to generate context via UnifiedContextManager: {e}")

        # FALLBACK: Fallback zu alter Methode falls UnifiedContextManager fehlschlägt
        if format_for_llm:
            return f"Error generating context: {str(e)}"
        else:
            return {
                "error": str(e),
                "generated_at": datetime.now().isoformat(),
                "fallback_mode": True
            }

get_context_overview(session_id=None, display=False) async

Detaillierte Übersicht des aktuellen Contexts mit Token-Counts und optionaler Display-Darstellung

Parameters:

Name	Type	Description	Default
session_id	str	Session ID für context (default: active_session)	None
display	bool	Ob die Übersicht im Terminal-Style angezeigt werden soll	False

Returns:

Name	Type	Description
dict	dict[str, Any]	Detaillierte Context-Übersicht mit Raw-Daten und Token-Counts

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

async def get_context_overview(self, session_id: str = None, display: bool = False) -> dict[str, Any]:
    """
    Detaillierte Übersicht des aktuellen Contexts mit Token-Counts und optionaler Display-Darstellung

    Args:
        session_id: Session ID für context (default: active_session)
        display: Ob die Übersicht im Terminal-Style angezeigt werden soll

    Returns:
        dict: Detaillierte Context-Übersicht mit Raw-Daten und Token-Counts
    """
    try:
        session_id = session_id or self.active_session or "default"

        # Token counting function
        def count_tokens(text: str) -> int:
            """Einfache Token-Approximation (4 chars ≈ 1 token für deutsche/englische Texte)"""
            try:
                from litellm import token_counter
                return token_counter(self.amd.fast_llm_model, text=text)
            except:
                pass
            return max(1, len(str(text)) // 4)

        context_overview = {
            "session_info": {
                "session_id": session_id,
                "agent_name": self.amd.name,
                "timestamp": datetime.now().isoformat(),
                "active_session": self.active_session,
                "is_running": self.is_running
            },
            "system_prompt": {},
            "meta_tools": {},
            "agent_tools": {},
            "mcp_tools": {},
            "variables": {},
            "system_history": {},
            "unified_context": {},
            "reasoning_context": {},
            "llm_tool_context": {},
            "token_summary": {}
        }

        # === SYSTEM PROMPT ANALYSIS ===
        system_message = self.amd.get_system_message_with_persona()
        context_overview["system_prompt"] = {
            "raw_data": system_message,
            "token_count": count_tokens(system_message),
            "components": {
                "base_message": self.amd.system_message,
                "persona_active": self.amd.persona is not None,
                "persona_name": self.amd.persona.name if self.amd.persona else None,
                "persona_integration": self.amd.persona.apply_method if self.amd.persona else None
            }
        }

        # === META TOOLS ANALYSIS ===
        if hasattr(self.task_flow, 'llm_reasoner') and hasattr(self.task_flow.llm_reasoner, 'meta_tools_registry'):
            meta_tools = self.task_flow.llm_reasoner.meta_tools_registry
        else:
            meta_tools = {}

        meta_tools_info = ""
        for tool_name, tool_info in meta_tools.items():
            tool_desc = tool_info.get("description", "No description")
            meta_tools_info += f"{tool_name}: {tool_desc}\n"

        # Standard Meta-Tools
        standard_meta_tools = [
            "internal_reasoning", "manage_internal_task_stack", "delegate_to_llm_tool_node",
            "create_and_execute_plan", "advance_outline_step", "write_to_variables",
            "read_from_variables", "direct_response"
        ]

        for meta_tool in standard_meta_tools:
            meta_tools_info += f"{meta_tool}: Built-in meta-tool for agent orchestration\n"

        context_overview["meta_tools"] = {
            "raw_data": meta_tools_info,
            "token_count": count_tokens(meta_tools_info),
            "count": len(meta_tools) + len(standard_meta_tools),
            "custom_meta_tools": list(meta_tools.keys()),
            "standard_meta_tools": standard_meta_tools
        }

        # === AGENT TOOLS ANALYSIS ===
        tools_info = ""
        tool_capabilities_text = ""

        for tool_name in self.shared.get("available_tools", []):
            tool_data = self._tool_registry.get(tool_name, {})
            description = tool_data.get("description", "No description")
            args_schema = tool_data.get("args_schema", "()")
            tools_info += f"{tool_name}{args_schema}: {description}\n"

            # Tool capabilities if available
            if tool_name in self._tool_capabilities:
                cap = self._tool_capabilities[tool_name]
                primary_function = cap.get("primary_function", "Unknown")
                use_cases = cap.get("use_cases", [])
                tool_capabilities_text += f"{tool_name}: {primary_function}\n"
                if use_cases:
                    tool_capabilities_text += f"  Use cases: {', '.join(use_cases[:3])}\n"

        context_overview["agent_tools"] = {
            "raw_data": tools_info,
            "capabilities_data": tool_capabilities_text,
            "token_count": count_tokens(tools_info + tool_capabilities_text),
            "count": len(self.shared.get("available_tools", [])),
            "analyzed_count": len(self._tool_capabilities),
            "tool_names": self.shared.get("available_tools", []),
            "intelligence_level": "high" if self._tool_capabilities else "basic"
        }

        # === MCP TOOLS ANALYSIS ===
        # Placeholder für MCP Tools (falls implementiert)
        mcp_tools_info = "No MCP tools currently active"
        if self.mcp_server:
            mcp_tools_info = f"MCP Server active: {getattr(self.mcp_server, 'name', 'Unknown')}"

        context_overview["mcp_tools"] = {
            "raw_data": mcp_tools_info,
            "token_count": count_tokens(mcp_tools_info),
            "server_active": bool(self.mcp_server),
            "server_name": getattr(self.mcp_server, 'name', None) if self.mcp_server else None
        }

        # === VARIABLES ANALYSIS ===
        variables_text = ""
        if self.variable_manager:
            variables_text = self.variable_manager.get_llm_variable_context()
        else:
            variables_text = "No variable manager available"

        context_overview["variables"] = {
            "raw_data": variables_text,
            "token_count": count_tokens(variables_text),
            "manager_available": bool(self.variable_manager),
            "total_scopes": len(self.variable_manager.scopes) if self.variable_manager else 0,
            "scope_names": list(self.variable_manager.scopes.keys()) if self.variable_manager else []
        }

        # === SYSTEM HISTORY ANALYSIS ===
        history_text = ""
        if self.context_manager and session_id in self.context_manager.session_managers:
            session = self.context_manager.session_managers[session_id]
            if hasattr(session, 'history'):
                history_count = len(session.history)
                history_text = f"Session History: {history_count} messages\n"

                # Recent messages preview
                for msg in session.history[-3:]:
                    role = msg.get('role', 'unknown')
                    content = msg.get('content', '')[:100] + "..." if len(
                        msg.get('content', '')) > 100 else msg.get('content', '')
                    timestamp = msg.get('timestamp', '')[:19]
                    history_text += f"[{timestamp}] {role}: {content}\n"
            elif isinstance(session, dict) and 'history' in session:
                history_count = len(session['history'])
                history_text = f"Fallback Session History: {history_count} messages"
        else:
            history_text = "No session history available"

        context_overview["system_history"] = {
            "raw_data": history_text,
            "token_count": count_tokens(history_text),
            "session_initialized": self.shared.get("session_initialized", False),
            "context_manager_available": bool(self.context_manager),
            "session_count": len(self.context_manager.session_managers) if self.context_manager else 0
        }

        # === UNIFIED CONTEXT ANALYSIS ===
        unified_context_text = ""
        try:
            unified_context = await self.context_manager.build_unified_context(session_id, "",
                                                                               "full") if self.context_manager else {}
            if unified_context:
                formatted_context = self.context_manager.get_formatted_context_for_llm(unified_context)
                unified_context_text = formatted_context
            else:
                unified_context_text = "No unified context available"
        except Exception as e:
            unified_context_text = f"Error building unified context: {str(e)}"

        context_overview["unified_context"] = {
            "raw_data": unified_context_text,
            "token_count": count_tokens(unified_context_text),
            "build_successful": "Error" not in unified_context_text,
            "manager_available": bool(self.context_manager)
        }

        # === REASONING CONTEXT ANALYSIS ===
        reasoning_context_text = ""
        if hasattr(self.task_flow, 'llm_reasoner') and hasattr(self.task_flow.llm_reasoner, 'reasoning_context'):
            reasoning_context = self.task_flow.llm_reasoner.reasoning_context
            reasoning_context_text = f"Reasoning Context: {len(reasoning_context)} entries\n"

            # Recent reasoning entries
            for entry in reasoning_context[-3:]:
                entry_type = entry.get('type', 'unknown')
                content = str(entry.get('content', ''))[:150] + "..." if len(
                    str(entry.get('content', ''))) > 150 else str(entry.get('content', ''))
                reasoning_context_text += f"  {entry_type}: {content}\n"
        else:
            reasoning_context_text = "No reasoning context available"

        context_overview["reasoning_context"] = {
            "raw_data": reasoning_context_text,
            "token_count": count_tokens(reasoning_context_text),
            "reasoner_available": hasattr(self.task_flow, 'llm_reasoner'),
            "context_entries": len(self.task_flow.llm_reasoner.reasoning_context) if hasattr(self.task_flow,
                                                                                             'llm_reasoner') and hasattr(
                self.task_flow.llm_reasoner, 'reasoning_context') else 0
        }

        # === LLM TOOL CONTEXT ANALYSIS ===
        llm_tool_context_text = ""
        if hasattr(self.task_flow, 'llm_tool_node'):
            llm_tool_context_text = f"LLM Tool Node available with max {self.task_flow.llm_tool_node.max_tool_calls} tool calls\n"
            if hasattr(self.task_flow.llm_tool_node, 'call_log'):
                call_log = self.task_flow.llm_tool_node.call_log
                llm_tool_context_text += f"Call log: {len(call_log)} entries\n"
        else:
            llm_tool_context_text = "No LLM Tool Node available"

        context_overview["llm_tool_context"] = {
            "raw_data": llm_tool_context_text,
            "token_count": count_tokens(llm_tool_context_text),
            "node_available": hasattr(self.task_flow, 'llm_tool_node'),
            "max_tool_calls": getattr(self.task_flow.llm_tool_node, 'max_tool_calls', 0) if hasattr(self.task_flow,
                                                                                                    'llm_tool_node') else 0
        }

        # === TOKEN SUMMARY ===
        total_tokens = sum([
            context_overview["system_prompt"]["token_count"],
            context_overview["meta_tools"]["token_count"],
            context_overview["agent_tools"]["token_count"],
            context_overview["mcp_tools"]["token_count"],
            context_overview["variables"]["token_count"],
            context_overview["system_history"]["token_count"],
            context_overview["unified_context"]["token_count"],
            context_overview["reasoning_context"]["token_count"],
            context_overview["llm_tool_context"]["token_count"]
        ])

        context_overview["token_summary"] = {
            "total_tokens": total_tokens,
            "breakdown": {
                "system_prompt": context_overview["system_prompt"]["token_count"],
                "meta_tools": context_overview["meta_tools"]["token_count"],
                "agent_tools": context_overview["agent_tools"]["token_count"],
                "mcp_tools": context_overview["mcp_tools"]["token_count"],
                "variables": context_overview["variables"]["token_count"],
                "system_history": context_overview["system_history"]["token_count"],
                "unified_context": context_overview["unified_context"]["token_count"],
                "reasoning_context": context_overview["reasoning_context"]["token_count"],
                "llm_tool_context": context_overview["llm_tool_context"]["token_count"]
            },
            "percentage_breakdown": {}
        }

        # Calculate percentages
        for component, token_count in context_overview["token_summary"]["breakdown"].items():
            percentage = (token_count / total_tokens * 100) if total_tokens > 0 else 0
            context_overview["token_summary"]["percentage_breakdown"][component] = round(percentage, 1)

        # === DISPLAY OUTPUT ===
        if display:
            await self._display_context_overview(context_overview)

        return context_overview

    except Exception as e:
        eprint(f"Error generating context overview: {e}")
        return {
            "error": str(e),
            "timestamp": datetime.now().isoformat(),
            "session_id": session_id
        }

get_context_statistics()

Get comprehensive context management statistics

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

def get_context_statistics(self) -> dict[str, Any]:
    """Get comprehensive context management statistics"""
    stats = {
        "context_system": "advanced_session_aware",
        "compression_threshold": 0.76,
        "max_tokens": getattr(self, 'max_input_tokens', 8000),
        "session_managers": {},
        "context_usage": {},
        "compression_stats": {}
    }

    # Session manager statistics
    session_managers = self.shared.get("session_managers", {})
    for name, manager in session_managers.items():
        stats["session_managers"][name] = {
            "history_length": len(manager.history if hasattr(manager, 'history') else (manager.get("history", []) if hasattr(manager, 'get') else [])),
            "max_length": manager.max_length if hasattr(manager, 'max_length') else manager.get("max_length", 0),
            "space_name": manager.space_name if hasattr(manager, 'space_name') else manager.get("space_name", "")
        }

    # Context node statistics if available
    if hasattr(self.task_flow, 'context_manager'):
        context_manager = self.task_flow.context_manager
        stats["compression_stats"] = {
            "compression_threshold": context_manager.compression_threshold,
            "max_tokens": context_manager.max_tokens,
            "active_sessions": len(context_manager.session_managers)
        }

    # LLM call statistics from enhanced node
    llm_stats = self.shared.get("llm_call_stats", {})
    if llm_stats:
        stats["context_usage"] = {
            "total_llm_calls": llm_stats.get("total_calls", 0),
            "context_compression_rate": llm_stats.get("context_compression_rate", 0.0),
            "average_context_tokens": llm_stats.get("context_tokens_used", 0) / max(llm_stats.get("total_calls", 1),
                                                                                    1)
        }

    return stats

get_format_quality_report()

Erhalte detaillierten Format-Qualitätsbericht

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

def get_format_quality_report(self) -> dict[str, Any]:
    """Erhalte detaillierten Format-Qualitätsbericht"""
    quality_assessment = self.shared.get("quality_assessment", {})

    if not quality_assessment:
        return {"status": "no_assessment", "message": "No recent quality assessment available"}

    quality_details = quality_assessment.get("quality_details", {})

    return {
        "overall_score": quality_details.get("total_score", 0.0),
        "format_adherence": quality_details.get("format_adherence", 0.0),
        "length_adherence": quality_details.get("length_adherence", 0.0),
        "content_quality": quality_details.get("base_quality", 0.0),
        "llm_assessment": quality_details.get("llm_assessment", 0.0),
        "suggestions": quality_assessment.get("suggestions", []),
        "assessment": quality_assessment.get("quality_assessment", "unknown"),
        "format_config_active": quality_details.get("format_config_used", False)
    }

get_session_storage_stats()

Get comprehensive session storage statistics

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

def get_session_storage_stats(self) -> dict[str, Any]:
    """Get comprehensive session storage statistics"""
    try:
        stats = {
            "context_manager_active": bool(self.context_manager),
            "total_sessions": 0,
            "session_details": {},
            "storage_summary": {
                "total_messages": 0,
                "context_snapshots": 0,
                "context_entries": 0,
                "regular_messages": 0
            }
        }

        if not self.context_manager:
            return stats

        stats["total_sessions"] = len(self.context_manager.session_managers)

        for session_id, session in self.context_manager.session_managers.items():
            session_stats = {
                "session_type": "chatsession" if hasattr(session, 'history') else "fallback",
                "message_count": 0,
                "context_snapshots": 0,
                "context_entries": 0,
                "regular_messages": 0,
                "storage_size_estimate": 0
            }

            if hasattr(session, 'history'):
                session_stats["message_count"] = len(session.history)

                for message in session.history:
                    content_size = len(str(message))
                    session_stats["storage_size_estimate"] += content_size

                    metadata = message.get("metadata", {})
                    if metadata.get("is_context_snapshot"):
                        session_stats["context_snapshots"] += 1
                    elif metadata.get("is_context_entry"):
                        session_stats["context_entries"] += 1
                    else:
                        session_stats["regular_messages"] += 1

            elif isinstance(session, dict) and 'history' in session:
                session_stats["message_count"] = len(session['history'])
                session_stats["regular_messages"] = len(session['history'])
                session_stats["storage_size_estimate"] = sum(len(str(msg)) for msg in session['history'])

            stats["session_details"][session_id] = session_stats

            # Update totals
            stats["storage_summary"]["total_messages"] += session_stats["message_count"]
            stats["storage_summary"]["context_snapshots"] += session_stats["context_snapshots"]
            stats["storage_summary"]["context_entries"] += session_stats["context_entries"]
            stats["storage_summary"]["regular_messages"] += session_stats["regular_messages"]

        # Estimate total storage size
        stats["storage_summary"]["estimated_total_size_kb"] = sum(
            details["storage_size_estimate"] for details in stats["session_details"].values()
        ) / 1024

        return stats

    except Exception as e:
        eprint(f"Failed to get session storage stats: {e}")
        return {"error": str(e)}

get_task_execution_summary() async

Erhalte detaillierte Zusammenfassung der Task-Ausführung

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

async def get_task_execution_summary(self) -> dict[str, Any]:
    """Erhalte detaillierte Zusammenfassung der Task-Ausführung"""
    tasks = self.shared.get("tasks", {})
    results_store = self.shared.get("results", {})

    summary = {
        "total_tasks": len(tasks),
        "completed_tasks": [],
        "failed_tasks": [],
        "task_types_used": {},
        "tools_used": [],
        "adaptations": self.shared.get("plan_adaptations", 0),
        "execution_timeline": []
    }

    for task_id, task in tasks.items():
        task_info = {
            "id": task_id,
            "type": task.type,
            "description": task.description,
            "status": task.status,
            "duration": None
        }

        if task.started_at and task.completed_at:
            duration = (task.completed_at - task.started_at).total_seconds()
            task_info["duration"] = duration

        if task.status == "completed":
            summary["completed_tasks"].append(task_info)
            if isinstance(task, ToolTask):
                summary["tools_used"].append(task.tool_name)
        elif task.status == "failed":
            task_info["error"] = task.error
            summary["failed_tasks"].append(task_info)

        # Task types counting
        task_type = task.type
        summary["task_types_used"][task_type] = summary["task_types_used"].get(task_type, 0) + 1

    return summary

get_tool_by_name(tool_name)

Get tool function by name

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

def get_tool_by_name(self, tool_name: str) -> Callable | None:
    """Get tool function by name"""
    return self._tool_registry.get(tool_name, {}).get("function")

get_variable(path, default=None)

Get variable using unified system

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

def get_variable(self, path: str, default=None):
    """Get variable using unified system"""
    return self.variable_manager.get(path, default)

get_variable_documentation()

Get comprehensive variable system documentation

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

def get_variable_documentation(self) -> str:
    """Get comprehensive variable system documentation"""
    docs = []
    docs.append("# Variable System Documentation\n")

    # Available scopes
    docs.append("## Available Scopes:")
    scope_info = self.variable_manager.get_scope_info()
    for scope_name, info in scope_info.items():
        docs.append(f"- `{scope_name}`: {info['type']} with {info.get('keys', 'N/A')} keys")

    docs.append("\n## Syntax Options:")
    docs.append("- `{{ variable.path }}` - Full path resolution")
    docs.append("- `{variable}` - Simple variable (no dots)")
    docs.append("- `$variable` - Shell-style variable")

    docs.append("\n## Example Usage:")
    docs.append("- `{{ results.task_1.data }}` - Get result from task_1")
    docs.append("- `{{ user.name }}` - Get user name")
    docs.append("- `{agent_name}` - Simple agent name")
    docs.append("- `$timestamp` - System timestamp")

    # Available variables
    docs.append("\n## Available Variables:")
    variables = self.variable_manager.get_available_variables()
    for scope_name, scope_vars in variables.items():
        docs.append(f"\n### {scope_name}:")
        for _var_name, var_info in scope_vars.items():
            docs.append(f"- `{var_info['path']}`: {var_info['preview']} ({var_info['type']})")

    return "\n".join(docs)

initialize_context_awareness() async

Enhanced context awareness with session management

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

async def initialize_context_awareness(self):
    """Enhanced context awareness with session management"""

    # Initialize session if not already done
    session_id = self.shared.get("session_id", self.active_session)
    if not self.shared.get("session_initialized"):
        await self.initialize_session_context(session_id)

    # Ensure tool capabilities are loaded
    # add tqdm prigress bar

    from tqdm import tqdm

    if hasattr(self.task_flow, 'llm_reasoner'):
        if "read_from_variables" not in self.shared["available_tools"] and hasattr(self.task_flow.llm_reasoner, '_execute_read_from_variables'):
            await self.add_tool(lambda scope, key, purpose: self.task_flow.llm_reasoner._execute_read_from_variables({"scope": scope, "key": key, "purpose": purpose}), "read_from_variables", "Read from variables")
        if "write_to_variables" not in self.shared["available_tools"] and hasattr(self.task_flow.llm_reasoner, '_execute_write_to_variables'):
            await self.add_tool(lambda scope, key, value, description: self.task_flow.llm_reasoner._execute_write_to_variables({"scope": scope, "key": key, "value": value, "description": description}), "write_to_variables", "Write to variables")

        if "internal_reasoning" not in self.shared["available_tools"] and hasattr(self.task_flow.llm_reasoner, '_execute_internal_reasoning'):
            async def internal_reasoning_tool(thought:str, thought_number:int, total_thoughts:int, next_thought_needed:bool, current_focus:str, key_insights:list[str], potential_issues:list[str], confidence_level:float):
                args = {
                    "thought": thought,
                    "thought_number": thought_number,
                    "total_thoughts": total_thoughts,
                    "next_thought_needed": next_thought_needed,
                    "current_focus": current_focus,
                    "key_insights": key_insights,
                    "potential_issues": potential_issues,
                    "confidence_level": confidence_level
                }
                return await self.task_flow.llm_reasoner._execute_internal_reasoning(args, self.shared)
            await self.add_tool(internal_reasoning_tool, "internal_reasoning", "Internal reasoning")

        if "manage_internal_task_stack" not in self.shared["available_tools"] and hasattr(self.task_flow.llm_reasoner, '_execute_manage_task_stack'):
            async def manage_internal_task_stack_tool(action:str, task_description:str, outline_step_ref:str):
                args = {
                    "action": action,
                    "task_description": task_description,
                    "outline_step_ref": outline_step_ref
                }
                return await self.task_flow.llm_reasoner._execute_manage_task_stack(args, self.shared)
            await self.add_tool(manage_internal_task_stack_tool, "manage_internal_task_stack", "Manage internal task stack")

        if "outline_step_completion" not in self.shared["available_tools"] and hasattr(self.task_flow.llm_reasoner, '_execute_outline_step_completion'):
            async def outline_step_completion_tool(step_completed:bool, completion_evidence:str, next_step_focus:str):
                args = {
                    "step_completed": step_completed,
                    "completion_evidence": completion_evidence,
                    "next_step_focus": next_step_focus
                }
                return await self.task_flow.llm_reasoner._execute_outline_step_completion(args, self.shared)
            await self.add_tool(outline_step_completion_tool, "outline_step_completion", "Outline step completion")


    registered_tools = set(self._tool_registry.keys())
    cached_capabilities = list(self._tool_capabilities.keys())  # Create a copy of
    for tool_name in cached_capabilities:
        if tool_name in self._tool_capabilities and tool_name not in registered_tools:
            del self._tool_capabilities[tool_name]
            iprint(f"Removed outdated capability for unavailable tool: {tool_name}")

    for tool_name in tqdm(self.shared["available_tools"], desc=f"Agent {self.amd.name} Analyzing Tools", unit="tool", colour="green", total=len(self.shared["available_tools"])):
        if tool_name not in self._tool_capabilities:
            tool_info = self._tool_registry.get(tool_name, {})
            description = tool_info.get("description", "No description")
            with Spinner(f"Analyzing tool {tool_name}"):
                await self._analyze_tool_capabilities(tool_name, description, tool_info.get("args_schema", "()"))

        if tool_name in self._tool_capabilities:
            function = self._tool_registry[tool_name]["function"]
            if not isinstance(self._tool_capabilities[tool_name], dict):
                self._tool_capabilities[tool_name] = {}
            self._tool_capabilities[tool_name]["args_schema"] = get_args_schema(function)

    # Set enhanced system context
    self.shared["system_context"] = {
        "capabilities_summary": self._build_capabilities_summary(),
        "tool_count": len(self.shared["available_tools"]),
        "analysis_loaded": len(self._tool_capabilities),
        "intelligence_level": "high" if self._tool_capabilities else "basic",
        "context_management": "advanced_session_aware",
        "session_managers": len(self.shared.get("session_managers", {})),
    }


    rprint("Advanced context awareness initialized with session management")

initialize_session_context(session_id='default', max_history=200) async

Vereinfachte Session-Initialisierung über UnifiedContextManager

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

async def initialize_session_context(self, session_id: str = "default", max_history: int = 200) -> bool:
    """Vereinfachte Session-Initialisierung über UnifiedContextManager"""
    try:
        # Delegation an UnifiedContextManager
        session = await self.context_manager.initialize_session(session_id, max_history)

        # Ensure Variable Manager integration
        if not self.context_manager.variable_manager:
            self.context_manager.variable_manager = self.variable_manager

        # Update shared state (minimal - primary data now in context_manager)
        self.shared["active_session_id"] = session_id
        self.shared["session_initialized"] = True

        # Legacy support: Keep session_managers reference in shared for backward compatibility
        self.shared["session_managers"] = self.context_manager.session_managers

        rprint(f"Session context initialized for {session_id} via UnifiedContextManager")
        return True

    except Exception as e:
        eprint(f"Session context initialization failed: {e}")
        import traceback
        print(traceback.format_exc())
        return False

list_available_checkpoints(max_age_hours=168)

List all available checkpoints with metadata

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

def list_available_checkpoints(self, max_age_hours: int = 168) -> list[dict[str, Any]]:  # Default 1 week
    """List all available checkpoints with metadata"""
    try:
        from toolboxv2 import get_app
        folder = str(get_app().data_dir) + '/Agents/checkpoint/' + self.amd.name

        if not os.path.exists(folder):
            return []

        checkpoints = []
        for file in os.listdir(folder):
            if file.endswith('.pkl') and file.startswith('agent_checkpoint_'):
                filepath = os.path.join(folder, file)
                try:
                    # Get file info
                    file_stat = os.stat(filepath)
                    file_size = file_stat.st_size
                    modified_time = datetime.fromtimestamp(file_stat.st_mtime)

                    # Extract timestamp from filename
                    timestamp_str = file.replace('agent_checkpoint_', '').replace('.pkl', '')
                    if timestamp_str == 'final_checkpoint':
                        checkpoint_time = modified_time
                        checkpoint_type = "final"
                    else:
                        checkpoint_time = datetime.strptime(timestamp_str, "%Y%m%d_%H%M%S")
                        checkpoint_type = "regular"

                    # Check age
                    age_hours = (datetime.now() - checkpoint_time).total_seconds() / 3600
                    if age_hours <= max_age_hours:

                        # Try to load checkpoint metadata without full loading
                        metadata = {}
                        try:
                            with open(filepath, 'rb') as f:
                                checkpoint = pickle.load(f)
                            metadata = {
                                "tasks_count": len(checkpoint.task_state) if checkpoint.task_state else 0,
                                "world_model_entries": len(checkpoint.world_model) if checkpoint.world_model else 0,
                                "session_id": checkpoint.metadata.get("session_id", "unknown") if hasattr(
                                    checkpoint, 'metadata') and checkpoint.metadata else "unknown",
                                "last_query": checkpoint.metadata.get("last_query", "unknown")[:100] if hasattr(
                                    checkpoint, 'metadata') and checkpoint.metadata else "unknown"
                            }
                        except:
                            metadata = {"load_error": True}

                        checkpoints.append({
                            "filepath": filepath,
                            "filename": file,
                            "checkpoint_type": checkpoint_type,
                            "timestamp": checkpoint_time.isoformat(),
                            "age_hours": round(age_hours, 1),
                            "file_size_kb": round(file_size / 1024, 1),
                            "metadata": metadata
                        })

                except Exception as e:
                    import traceback
                    print(traceback.format_exc())
                    wprint(f"Could not analyze checkpoint file {file}: {e}")
                    continue

        # Sort by timestamp (newest first)
        checkpoints.sort(key=lambda x: x["timestamp"], reverse=True)

        return checkpoints

    except Exception as e:
        import traceback
        print(traceback.format_exc())
        eprint(f"Failed to list checkpoints: {e}")
        return []

load_context_from_session(session_id, context_type='full') async

Load context from ChatSession storage

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

async def load_context_from_session(self, session_id: str, context_type: str = "full") -> dict[str, Any]:
    """Load context from ChatSession storage"""
    try:
        if not self.context_manager:
            return {"error": "Context manager not available"}

        session = self.context_manager.session_managers.get(session_id)
        if not session:
            return {"error": f"Session {session_id} not found"}

        # Search for context snapshots in session history
        context_snapshots = []

        if hasattr(session, 'history'):
            for message in reversed(session.history):  # Search from newest
                if (message.get("role") == "system" and
                    message.get("metadata", {}).get("is_context_snapshot") and
                    message.get("metadata", {}).get("context_type") == context_type):

                    try:
                        # Extract context data
                        content = message.get("content", "")
                        if content.startswith(f"[CONTEXT_SNAPSHOT_{context_type.upper()}]"):
                            json_data = content.replace(f"[CONTEXT_SNAPSHOT_{context_type.upper()}] ", "")
                            context_data = json.loads(json_data)
                            context_snapshots.append({
                                "context": context_data,
                                "timestamp": message.get("timestamp"),
                                "metadata": message.get("metadata", {})
                            })
                    except Exception as e:
                        wprint(f"Failed to parse context snapshot: {e}")

        if context_snapshots:
            # Return most recent context snapshot
            latest_context = context_snapshots[0]
            rprint(f"Loaded context snapshot from session {session_id} (timestamp: {latest_context['timestamp']})")
            return latest_context["context"]
        else:
            return {"error": f"No context snapshots of type '{context_type}' found in session {session_id}"}

    except Exception as e:
        eprint(f"Failed to load context from session: {e}")
        return {"error": str(e)}

load_latest_checkpoint(auto_restore_history=True, max_age_hours=24) async

Vereinfachtes Checkpoint-Laden mit automatischer History-Wiederherstellung

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

async def load_latest_checkpoint(self, auto_restore_history: bool = True, max_age_hours: int = 24) -> dict[
    str, Any]:
    """Vereinfachtes Checkpoint-Laden mit automatischer History-Wiederherstellung"""
    try:
        from toolboxv2 import get_app
        folder = str(get_app().data_dir) + '/Agents/checkpoint/' + self.amd.name

        if not os.path.exists(folder):
            return {"success": False, "error": "Kein Checkpoint-Verzeichnis gefunden"}

        # Finde neuesten Checkpoint
        checkpoint_files = []
        for file in os.listdir(folder):
            if file.endswith('.pkl') and file.startswith('agent_checkpoint_'):
                filepath = os.path.join(folder, file)
                try:
                    timestamp_str = file.replace('agent_checkpoint_', '').replace('.pkl', '')
                    if timestamp_str == 'final_checkpoint':
                        file_time = datetime.fromtimestamp(os.path.getmtime(filepath))
                    else:
                        file_time = datetime.strptime(timestamp_str, "%Y%m%d_%H%M%S")

                    age_hours = (datetime.now() - file_time).total_seconds() / 3600
                    if age_hours <= max_age_hours:
                        checkpoint_files.append((filepath, file_time, age_hours))
                except Exception:
                    continue

        if not checkpoint_files:
            return {"success": False, "error": f"Keine gültigen Checkpoints in {max_age_hours} Stunden gefunden"}

        # Lade neuesten Checkpoint
        checkpoint_files.sort(key=lambda x: x[1], reverse=True)
        latest_checkpoint_path, latest_timestamp, age_hours = checkpoint_files[0]

        rprint(f"Lade Checkpoint: {latest_checkpoint_path} (Alter: {age_hours:.1f}h)")

        with open(latest_checkpoint_path, 'rb') as f:
            checkpoint: AgentCheckpoint = pickle.load(f)

        # Stelle Agent-Status wieder her
        restore_stats = await self._restore_from_checkpoint_simplified(checkpoint, auto_restore_history)

        # Re-initialisiere Kontext-Awareness
        await self.initialize_context_awareness()

        return {
            "success": True,
            "checkpoint_file": latest_checkpoint_path,
            "checkpoint_age_hours": age_hours,
            "checkpoint_timestamp": latest_timestamp.isoformat(),
            "available_checkpoints": len(checkpoint_files),
            "restore_stats": restore_stats
        }

    except Exception as e:
        eprint(f"Checkpoint-Laden fehlgeschlagen: {e}")
        import traceback
        print(traceback.format_exc())
        return {"success": False, "error": str(e)}

pause() async

Pause agent execution

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

async def pause(self) -> bool:
    """Pause agent execution"""
    if not self.is_running:
        return False

    self.is_paused = True
    self.shared["system_status"] = "paused"

    # Create checkpoint
    checkpoint = await self._create_checkpoint()
    await self._save_checkpoint(checkpoint)

    rprint("Agent execution paused")
    return True

resume() async

Resume agent execution

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

async def resume(self) -> bool:
    """Resume agent execution"""
    if not self.is_paused:
        return False

    self.is_paused = False
    self.shared["system_status"] = "running"

    rprint("Agent execution resumed")
    return True

save_context_to_file(session_id=None) async

Save current context to file

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

async def save_context_to_file(self, session_id: str = None) -> bool:
    """Save current context to file"""
    try:
        context = await self.get_context(session_id=session_id, format_for_llm=False)

        filepath = self._get_context_path(session_id)

        with open(filepath, 'w', encoding='utf-8') as f:
            json.dump(context, f, indent=2, ensure_ascii=False, default=str)

        rprint(f"Context saved to: {filepath}")
        return True

    except Exception as e:
        eprint(f"Failed to save context: {e}")
        return False

save_context_to_session(session_id=None, context_type='full') async

Save current context to ChatSession for persistent storage

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

async def save_context_to_session(self, session_id: str = None, context_type: str = "full") -> bool:
    """Save current context to ChatSession for persistent storage"""
    try:
        session_id = session_id or self.shared.get("session_id", "default")

        if not self.context_manager:
            eprint("Context manager not available")
            return False

        # Build comprehensive context
        unified_context = await self.context_manager.build_unified_context(session_id, None, context_type)

        # Create context message for session storage
        context_message = {
            "role": "system",
            "content": f"[CONTEXT_SNAPSHOT_{context_type.upper()}] " + json.dumps(unified_context, default=str),
            "timestamp": datetime.now().isoformat(),
            "context_type": context_type,
            "metadata": {
                "is_context_snapshot": True,
                "context_version": "2.0",
                "agent_name": self.amd.name,
                "session_stats": unified_context.get("session_stats", {}),
                "variables_count": len(unified_context.get("variables", {}).get("recent_results", [])),
                "execution_state": unified_context.get("execution_state", {}).get("system_status", "unknown")
            }
        }

        # Store in session
        await self.context_manager.add_interaction(
            session_id,
            "system",
            context_message["content"],
            metadata=context_message["metadata"]
        )

        rprint(f"Context snapshot saved to session {session_id} (type: {context_type})")
        return True

    except Exception as e:
        eprint(f"Failed to save context to session: {e}")
        return False

set_persona(name, style='professional', tone='friendly', personality_traits=None, apply_method='system_prompt', integration_level='light', custom_instructions='')

Set agent persona mit erweiterten Konfigurationsmöglichkeiten

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

def set_persona(self, name: str, style: str = "professional", tone: str = "friendly",
                personality_traits: list[str] = None, apply_method: str = "system_prompt",
                integration_level: str = "light", custom_instructions: str = ""):
    """Set agent persona mit erweiterten Konfigurationsmöglichkeiten"""
    if personality_traits is None:
        personality_traits = ["helpful", "concise"]

    self.amd.persona = PersonaConfig(
        name=name,
        style=style,
        tone=tone,
        personality_traits=personality_traits,
        custom_instructions=custom_instructions,
        apply_method=apply_method,
        integration_level=integration_level
    )

    rprint(f"Persona set: {name} ({style}, {tone}) - Method: {apply_method}, Level: {integration_level}")

set_response_format(response_format, text_length, custom_instructions='', quality_threshold=0.7)

Dynamische Format- und Längen-Konfiguration

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

def set_response_format(
    self,
    response_format: str,
    text_length: str,
    custom_instructions: str = "",
    quality_threshold: float = 0.7
):
    """Dynamische Format- und Längen-Konfiguration"""

    # Validiere Eingaben
    try:
        ResponseFormat(response_format)
        TextLength(text_length)
    except ValueError:
        available_formats = [f.value for f in ResponseFormat]
        available_lengths = [l.value for l in TextLength]
        raise ValueError(
            f"Invalid format or length. "
            f"Available formats: {available_formats}. "
            f"Available lengths: {available_lengths}"
        )

    # Erstelle oder aktualisiere Persona
    if not self.amd.persona:
        self.amd.persona = PersonaConfig(name="Assistant")

    # Erstelle Format-Konfiguration
    format_config = FormatConfig(
        response_format=ResponseFormat(response_format),
        text_length=TextLength(text_length),
        custom_instructions=custom_instructions,
        quality_threshold=quality_threshold
    )

    self.amd.persona.format_config = format_config

    # Aktualisiere Personality Traits mit Format-Hinweisen
    self._update_persona_with_format(response_format, text_length)

    # Update shared state
    self.shared["persona_config"] = self.amd.persona
    self.shared["format_config"] = format_config

    rprint(f"Response format set: {response_format}, length: {text_length}")

set_variable(path, value)

Set variable using unified system

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

def set_variable(self, path: str, value: Any):
    """Set variable using unified system"""
    self.variable_manager.set(path, value)

setup_a2a_server(host='0.0.0.0', port=5000, **kwargs)

Setup A2A server for bidirectional communication

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

def setup_a2a_server(self, host: str = "0.0.0.0", port: int = 5000, **kwargs):
    """Setup A2A server for bidirectional communication"""
    if not A2A_AVAILABLE:
        wprint("A2A not available, cannot setup server")
        return

    try:
        self.a2a_server = A2AServer(
            host=host,
            port=port,
            agent_card=AgentCard(
                name=self.amd.name,
                description="Production-ready PocketFlow agent",
                version="1.0.0"
            ),
            **kwargs
        )

        # Register agent methods
        @self.a2a_server.route("/run")
        async def handle_run(request_data):
            query = request_data.get("query", "")
            session_id = request_data.get("session_id", "a2a_session")

            response = await self.a_run(query, session_id=session_id)
            return {"response": response}

        rprint(f"A2A server setup on {host}:{port}")

    except Exception as e:
        eprint(f"Failed to setup A2A server: {e}")

setup_mcp_server(host='0.0.0.0', port=8000, name=None, **kwargs)

Setup MCP server

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

def setup_mcp_server(self, host: str = "0.0.0.0", port: int = 8000, name: str = None, **kwargs):
    """Setup MCP server"""
    if not MCP_AVAILABLE:
        wprint("MCP not available, cannot setup server")
        return

    try:
        server_name = name or f"{self.amd.name}_MCP"
        self.mcp_server = FastMCP(server_name)

        # Register agent as MCP tool
        @self.mcp_server.tool()
        async def agent_run(query: str, session_id: str = "mcp_session") -> str:
            """Execute agent with given query"""
            return await self.a_run(query, session_id=session_id)

        rprint(f"MCP server setup: {server_name}")

    except Exception as e:
        eprint(f"Failed to setup MCP server: {e}")

start_servers() async

Start all configured servers

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

async def start_servers(self):
    """Start all configured servers"""
    tasks = []

    if self.a2a_server:
        tasks.append(asyncio.create_task(self.a2a_server.start()))

    if self.mcp_server:
        tasks.append(asyncio.create_task(self.mcp_server.run()))

    if tasks:
        rprint(f"Starting {len(tasks)} servers...")
        await asyncio.gather(*tasks, return_exceptions=True)

status(pretty_print=False) async

Get comprehensive agent status with optional pretty printing

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

async def status(self, pretty_print: bool = False) -> dict[str, Any] | str:
    """Get comprehensive agent status with optional pretty printing"""

    # Core status information
    base_status = {
        "agent_info": {
            "name": self.amd.name,
            "version": "2.0",
            "type": "FlowAgent"
        },
        "runtime_status": {
            "status": self.shared.get("system_status", "idle"),
            "is_running": self.is_running,
            "is_paused": self.is_paused,
            "uptime_seconds": (datetime.now() - getattr(self, '_start_time', datetime.now())).total_seconds()
        },
        "task_execution": {
            "total_tasks": len(self.shared.get("tasks", {})),
            "active_tasks": len([t for t in self.shared.get("tasks", {}).values() if t.status == "running"]),
            "completed_tasks": len([t for t in self.shared.get("tasks", {}).values() if t.status == "completed"]),
            "failed_tasks": len([t for t in self.shared.get("tasks", {}).values() if t.status == "failed"]),
            "plan_adaptations": self.shared.get("plan_adaptations", 0)
        },
        "conversation": {
            "turns": len(self.shared.get("conversation_history", [])),
            "session_id": self.shared.get("session_id", self.active_session),
            "current_user": self.shared.get("user_id"),
            "last_query": self.shared.get("current_query", "")[:100] + "..." if len(
                self.shared.get("current_query", "")) > 100 else self.shared.get("current_query", "")
        },
        "capabilities": {
            "available_tools": len(self.shared.get("available_tools", [])),
            "tool_names": list(self.shared.get("available_tools", [])),
            "analyzed_tools": len(self._tool_capabilities),
            "world_model_size": len(self.shared.get("world_model", {})),
            "intelligence_level": "high" if self._tool_capabilities else "basic"
        },
        "memory_context": {
            "session_initialized": self.shared.get("session_initialized", False),
            "session_managers": len(self.shared.get("session_managers", {})),
            "context_system": "advanced_session_aware" if self.shared.get("session_initialized") else "basic",
            "variable_scopes": len(self.variable_manager.get_scope_info()) if hasattr(self,
                                                                                      'variable_manager') else 0
        },
        "performance": {
            "total_cost": self.total_cost,
            "checkpoint_enabled": self.enable_pause_resume,
            "last_checkpoint": self.last_checkpoint.isoformat() if self.last_checkpoint else None,
            "max_parallel_tasks": self.max_parallel_tasks
        },
        "servers": {
            "a2a_server": self.a2a_server is not None,
            "mcp_server": self.mcp_server is not None,
            "server_count": sum([self.a2a_server is not None, self.mcp_server is not None])
        },
        "configuration": {
            "fast_llm_model": self.amd.fast_llm_model,
            "complex_llm_model": self.amd.complex_llm_model,
            "use_fast_response": getattr(self.amd, 'use_fast_response', False),
            "max_input_tokens": getattr(self.amd, 'max_input_tokens', 8000),
            "persona_configured": self.amd.persona is not None,
            "format_config": bool(getattr(self.amd.persona, 'format_config', None)) if self.amd.persona else False
        }
    }

    # Add detailed execution summary if tasks exist
    tasks = self.shared.get("tasks", {})
    if tasks:
        task_types_used = {}
        tools_used = []
        execution_timeline = []

        for task_id, task in tasks.items():
            # Count task types
            task_type = getattr(task, 'type', 'unknown')
            task_types_used[task_type] = task_types_used.get(task_type, 0) + 1

            # Collect tools used
            if hasattr(task, 'tool_name') and task.tool_name:
                tools_used.append(task.tool_name)

            # Timeline info
            if hasattr(task, 'started_at') and task.started_at:
                timeline_entry = {
                    "task_id": task_id,
                    "type": task_type,
                    "started": task.started_at.isoformat(),
                    "status": getattr(task, 'status', 'unknown')
                }
                if hasattr(task, 'completed_at') and task.completed_at:
                    timeline_entry["completed"] = task.completed_at.isoformat()
                    timeline_entry["duration"] = (task.completed_at - task.started_at).total_seconds()
                execution_timeline.append(timeline_entry)

        base_status["task_execution"].update({
            "task_types_used": task_types_used,
            "tools_used": list(set(tools_used)),
            "execution_timeline": execution_timeline[-5:]  # Last 5 tasks
        })

    # Add context statistics
    if hasattr(self.task_flow, 'context_manager'):
        context_manager = self.task_flow.context_manager
        base_status["memory_context"].update({
            "compression_threshold": context_manager.compression_threshold,
            "max_tokens": context_manager.max_tokens,
            "active_context_sessions": len(getattr(context_manager, 'session_managers', {}))
        })

    # Add variable system info
    if hasattr(self, 'variable_manager'):
        available_vars = self.variable_manager.get_available_variables()
        scope_info = self.variable_manager.get_scope_info()

        base_status["variable_system"] = {
            "total_scopes": len(scope_info),
            "scope_names": list(scope_info.keys()),
            "total_variables": sum(len(vars) for vars in available_vars.values()),
            "scope_details": {
                scope: {"type": info["type"], "variables": len(available_vars.get(scope, {}))}
                for scope, info in scope_info.items()
            }
        }

    # Add format quality info if available
    quality_assessment = self.shared.get("quality_assessment", {})
    if quality_assessment:
        quality_details = quality_assessment.get("quality_details", {})
        base_status["format_quality"] = {
            "overall_score": quality_details.get("total_score", 0.0),
            "format_adherence": quality_details.get("format_adherence", 0.0),
            "length_adherence": quality_details.get("length_adherence", 0.0),
            "content_quality": quality_details.get("base_quality", 0.0),
            "assessment": quality_assessment.get("quality_assessment", "unknown"),
            "has_suggestions": bool(quality_assessment.get("suggestions", []))
        }

    # Add LLM usage statistics
    llm_stats = self.shared.get("llm_call_stats", {})
    if llm_stats:
        base_status["llm_usage"] = {
            "total_calls": llm_stats.get("total_calls", 0),
            "context_compression_rate": llm_stats.get("context_compression_rate", 0.0),
            "average_context_tokens": llm_stats.get("context_tokens_used", 0) / max(llm_stats.get("total_calls", 1),
                                                                                    1),
            "total_tokens_used": llm_stats.get("total_tokens_used", 0)
        }

    # Add timestamp
    base_status["timestamp"] = datetime.now().isoformat()

    base_status["context_statistic"] = self.get_context_statistics()
    if not pretty_print:
        base_status["agent_context"] = await self.get_context_overview()
        return base_status

    # Pretty print using EnhancedVerboseOutput
    try:
        from toolboxv2.mods.isaa.extras.verbose_output import EnhancedVerboseOutput
        verbose_output = EnhancedVerboseOutput(verbose=True)

        # Header
        verbose_output.log_header(f"Agent Status: {base_status['agent_info']['name']}")

        # Runtime Status
        status_color = {
            "running": "SUCCESS",
            "paused": "WARNING",
            "idle": "INFO",
            "error": "ERROR"
        }.get(base_status["runtime_status"]["status"], "INFO")

        getattr(verbose_output, f"print_{status_color.lower()}")(
            f"Status: {base_status['runtime_status']['status'].upper()}"
        )

        # Task Execution Summary
        task_exec = base_status["task_execution"]
        if task_exec["total_tasks"] > 0:
            verbose_output.formatter.print_section(
                "Task Execution",
                f"Total: {task_exec['total_tasks']} | "
                f"Completed: {task_exec['completed_tasks']} | "
                f"Failed: {task_exec['failed_tasks']} | "
                f"Active: {task_exec['active_tasks']}\n"
                f"Adaptations: {task_exec['plan_adaptations']}"
            )

            if task_exec.get("tools_used"):
                verbose_output.formatter.print_section(
                    "Tools Used",
                    ", ".join(task_exec["tools_used"])
                )

        # Capabilities
        caps = base_status["capabilities"]
        verbose_output.formatter.print_section(
            "Capabilities",
            f"Intelligence Level: {caps['intelligence_level']}\n"
            f"Available Tools: {caps['available_tools']}\n"
            f"Analyzed Tools: {caps['analyzed_tools']}\n"
            f"World Model Size: {caps['world_model_size']}"
        )

        # Memory & Context
        memory = base_status["memory_context"]
        verbose_output.formatter.print_section(
            "Memory & Context",
            f"Context System: {memory['context_system']}\n"
            f"Session Managers: {memory['session_managers']}\n"
            f"Variable Scopes: {memory['variable_scopes']}\n"
            f"Session Initialized: {memory['session_initialized']}"
        )

        # Context Statistics
        stats = base_status["context_statistic"]
        verbose_output.formatter.print_section(
            "Context & Stats",
            f"Compression Stats: {stats['compression_stats']}\n"
            f"Session Usage: {stats['context_usage']}\n"
            f"Session Managers: {stats['session_managers']}\n"
        )

        # Configuration
        config = base_status["configuration"]
        verbose_output.formatter.print_section(
            "Configuration",
            f"Fast LLM: {config['fast_llm_model']}\n"
            f"Complex LLM: {config['complex_llm_model']}\n"
            f"Max Tokens: {config['max_input_tokens']}\n"
            f"Persona: {'Configured' if config['persona_configured'] else 'Default'}\n"
            f"Format Config: {'Active' if config['format_config'] else 'None'}"
        )

        # Performance
        perf = base_status["performance"]
        verbose_output.formatter.print_section(
            "Performance",
            f"Total Cost: ${perf['total_cost']:.4f}\n"
            f"Checkpointing: {'Enabled' if perf['checkpoint_enabled'] else 'Disabled'}\n"
            f"Max Parallel Tasks: {perf['max_parallel_tasks']}\n"
            f"Last Checkpoint: {perf['last_checkpoint'] or 'None'}"
        )

        # Variable System Details
        if "variable_system" in base_status:
            var_sys = base_status["variable_system"]
            scope_details = []
            for scope, details in var_sys["scope_details"].items():
                scope_details.append(f"{scope}: {details['variables']} variables ({details['type']})")

            verbose_output.formatter.print_section(
                "Variable System",
                f"Total Scopes: {var_sys['total_scopes']}\n"
                f"Total Variables: {var_sys['total_variables']}\n" +
                "\n".join(scope_details)
            )

        # Format Quality
        if "format_quality" in base_status:
            quality = base_status["format_quality"]
            verbose_output.formatter.print_section(
                "Format Quality",
                f"Overall Score: {quality['overall_score']:.2f}\n"
                f"Format Adherence: {quality['format_adherence']:.2f}\n"
                f"Length Adherence: {quality['length_adherence']:.2f}\n"
                f"Content Quality: {quality['content_quality']:.2f}\n"
                f"Assessment: {quality['assessment']}"
            )

        # LLM Usage
        if "llm_usage" in base_status:
            llm = base_status["llm_usage"]
            verbose_output.formatter.print_section(
                "LLM Usage Statistics",
                f"Total Calls: {llm['total_calls']}\n"
                f"Avg Context Tokens: {llm['average_context_tokens']:.1f}\n"
                f"Total Tokens: {llm['total_tokens_used']}\n"
                f"Compression Rate: {llm['context_compression_rate']:.2%}"
            )

        # Servers
        servers = base_status["servers"]
        if servers["server_count"] > 0:
            server_status = []
            if servers["a2a_server"]:
                server_status.append("A2A Server: Active")
            if servers["mcp_server"]:
                server_status.append("MCP Server: Active")

            verbose_output.formatter.print_section(
                "Servers",
                "\n".join(server_status)
            )

        verbose_output.print_separator()
        await self.get_context_overview(display=True)
        verbose_output.print_separator()
        verbose_output.print_info(f"Status generated at: {base_status['timestamp']}")

        return "Status printed above"

    except Exception:
        # Fallback to JSON if pretty print fails
        import json
        return json.dumps(base_status, indent=2, default=str)

unbind(preserve_shared_data=False)

Unbind this agent from any binding configuration.

Parameters:

Name	Type	Description	Default
preserve_shared_data	bool	Whether to preserve shared data in the agent after unbinding	False

Returns:

Name	Type	Description
dict		Unbinding result with statistics

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

def unbind(self, preserve_shared_data: bool = False):
    """
    Unbind this agent from any binding configuration.

    Args:
        preserve_shared_data: Whether to preserve shared data in the agent after unbinding

    Returns:
        dict: Unbinding result with statistics
    """
    if not self.shared.get('is_bound', False):
        return {
            'success': False,
            'message': f"Agent {self.amd.name} is not currently bound to any other agents"
        }

    binding_config = self.shared.get('binding_config')
    if not binding_config:
        return {
            'success': False,
            'message': "No binding configuration found"
        }

    binding_id = binding_config['binding_id']
    bound_agents = binding_config['agents']

    unbind_stats = {
        'binding_id': binding_id,
        'agents_affected': [],
        'shared_data_preserved': preserve_shared_data,
        'private_data_restored': False,
        'unbind_timestamp': datetime.now().isoformat()
    }

    try:
        # Restore original managers for this agent
        if hasattr(self, '_original_managers'):
            original = self._original_managers

            if preserve_shared_data:
                # Merge current shared data with original data
                if isinstance(original['world_model'], dict):
                    original['world_model'].update(self.world_model)
                if isinstance(original['shared'], dict):
                    original['shared'].update({k: v for k, v in self.shared.items()
                                               if k not in ['binding_config', 'is_bound', 'binding_id',
                                                            'bound_agents']})

            # Restore original variable manager
            self.variable_manager = original['variable_manager']
            self.world_model = original['world_model']
            self.shared = original['shared']

            # Update context manager
            if hasattr(self, 'context_manager') and self.context_manager:
                self.context_manager.variable_manager = self.variable_manager

            unbind_stats['private_data_restored'] = True
            del self._original_managers

        # Clean up binding state
        self.shared.pop('binding_config', None)
        self.shared.pop('is_bound', None)
        self.shared.pop('binding_id', None)
        self.shared.pop('bound_agents', None)

        # Update binding configuration to remove this agent
        remaining_agents = [agent for agent in bound_agents if agent != self]
        if remaining_agents:
            # Update binding config for remaining agents
            binding_config['agents'] = remaining_agents
            for agent in remaining_agents:
                if hasattr(agent, 'shared') and agent.shared.get('is_bound'):
                    agent.shared['bound_agents'] = [a.amd.name for a in remaining_agents]

        unbind_stats['agents_affected'] = [agent.amd.name for agent in bound_agents]

        # Clean up sync handler if this was the last agent
        if len(remaining_agents) <= 1:
            sync_handler = binding_config.get('sync_handler')
            if sync_handler and hasattr(sync_handler, 'cleanup'):
                sync_handler.cleanup()

        rprint(f"Agent {self.amd.name} successfully unbound from binding {binding_id}")
        rprint(f"Shared data preserved: {preserve_shared_data}")

        return {
            'success': True,
            'stats': unbind_stats,
            'message': f"Agent {self.amd.name} unbound successfully"
        }

    except Exception as e:
        eprint(f"Error during unbinding: {e}")
        return {
            'success': False,
            'error': str(e),
            'stats': unbind_stats
        }

FormatConfig dataclass

Konfiguration für Response-Format und -Länge

Source code in toolboxv2/mods/isaa/base/Agent/types.py

@dataclass
class FormatConfig:
    """Konfiguration für Response-Format und -Länge"""
    response_format: ResponseFormat = ResponseFormat.FREE_TEXT
    text_length: TextLength = TextLength.CHAT_CONVERSATION
    custom_instructions: str = ""
    strict_format_adherence: bool = True
    quality_threshold: float = 0.7

    def get_format_instructions(self) -> str:
        """Generiere Format-spezifische Anweisungen"""
        format_instructions = {
            ResponseFormat.FREE_TEXT: "Use natural continuous text without special formatting.",
            ResponseFormat.WITH_TABLES: "Integrate tables for structured data representation. Use Markdown tables.",
            ResponseFormat.WITH_BULLET_POINTS: "Structure information with bullet points (•, -, *) for better readability.",
            ResponseFormat.WITH_LISTS: "Use numbered and unnumbered lists to organize content.",
            ResponseFormat.TEXT_ONLY: "Plain text only without formatting, symbols, or structural elements.",
            ResponseFormat.MD_TEXT: "Full Markdown formatting with headings, code blocks, links, etc.",
            ResponseFormat.YAML_TEXT: "Structure responses in YAML format for machine-readable output.",
            ResponseFormat.JSON_TEXT: "Format responses as a JSON structure for API integration.",
            ResponseFormat.PSEUDO_CODE: "Use pseudocode structure for algorithmic or logical explanations.",
            ResponseFormat.CODE_STRUCTURE: "Structure like code with indentation, comments, and logical blocks."
        }
        return format_instructions.get(self.response_format, "Standard-Formatierung.")

    def get_length_instructions(self) -> str:
        """Generiere Längen-spezifische Anweisungen"""
        length_instructions = {
            TextLength.MINI_CHAT: "Very short, concise answers (1–2 sentences, max 50 words). Chat style.",
            TextLength.CHAT_CONVERSATION: "Moderate conversation length (2–4 sentences, 50–150 words). Natural conversational style.",
            TextLength.TABLE_CONVERSATION: "Structured, tabular presentation with compact explanations (100–250 words).",
            TextLength.DETAILED_INDEPTH: "Comprehensive, detailed explanations (300–800 words) with depth and context.",
            TextLength.PHD_LEVEL: "Academic depth with extensive explanations (800+ words), references, and technical terminology."
        }
        return length_instructions.get(self.text_length, "Standard-Länge.")

    def get_combined_instructions(self) -> str:
        """Kombiniere Format- und Längen-Anweisungen"""
        instructions = []
        instructions.append("## Format-Anforderungen:")
        instructions.append(self.get_format_instructions())
        instructions.append("\n## Längen-Anforderungen:")
        instructions.append(self.get_length_instructions())

        if self.custom_instructions:
            instructions.append("\n## Zusätzliche Anweisungen:")
            instructions.append(self.custom_instructions)

        if self.strict_format_adherence:
            instructions.append("\n## ATTENTION: STRICT FORMAT ADHERENCE REQUIRED!")

        return "\n".join(instructions)

    def get_expected_word_range(self) -> tuple[int, int]:
        """Erwartete Wortanzahl für Qualitätsbewertung"""
        ranges = {
            TextLength.MINI_CHAT: (10, 50),
            TextLength.CHAT_CONVERSATION: (50, 150),
            TextLength.TABLE_CONVERSATION: (100, 250),
            TextLength.DETAILED_INDEPTH: (300, 800),
            TextLength.PHD_LEVEL: (800, 2000)
        }
        return ranges.get(self.text_length, (50, 200))

get_combined_instructions()

Kombiniere Format- und Längen-Anweisungen

Source code in toolboxv2/mods/isaa/base/Agent/types.py

def get_combined_instructions(self) -> str:
    """Kombiniere Format- und Längen-Anweisungen"""
    instructions = []
    instructions.append("## Format-Anforderungen:")
    instructions.append(self.get_format_instructions())
    instructions.append("\n## Längen-Anforderungen:")
    instructions.append(self.get_length_instructions())

    if self.custom_instructions:
        instructions.append("\n## Zusätzliche Anweisungen:")
        instructions.append(self.custom_instructions)

    if self.strict_format_adherence:
        instructions.append("\n## ATTENTION: STRICT FORMAT ADHERENCE REQUIRED!")

    return "\n".join(instructions)

get_expected_word_range()

Erwartete Wortanzahl für Qualitätsbewertung

Source code in toolboxv2/mods/isaa/base/Agent/types.py

def get_expected_word_range(self) -> tuple[int, int]:
    """Erwartete Wortanzahl für Qualitätsbewertung"""
    ranges = {
        TextLength.MINI_CHAT: (10, 50),
        TextLength.CHAT_CONVERSATION: (50, 150),
        TextLength.TABLE_CONVERSATION: (100, 250),
        TextLength.DETAILED_INDEPTH: (300, 800),
        TextLength.PHD_LEVEL: (800, 2000)
    }
    return ranges.get(self.text_length, (50, 200))

get_format_instructions()

Generiere Format-spezifische Anweisungen

Source code in toolboxv2/mods/isaa/base/Agent/types.py

def get_format_instructions(self) -> str:
    """Generiere Format-spezifische Anweisungen"""
    format_instructions = {
        ResponseFormat.FREE_TEXT: "Use natural continuous text without special formatting.",
        ResponseFormat.WITH_TABLES: "Integrate tables for structured data representation. Use Markdown tables.",
        ResponseFormat.WITH_BULLET_POINTS: "Structure information with bullet points (•, -, *) for better readability.",
        ResponseFormat.WITH_LISTS: "Use numbered and unnumbered lists to organize content.",
        ResponseFormat.TEXT_ONLY: "Plain text only without formatting, symbols, or structural elements.",
        ResponseFormat.MD_TEXT: "Full Markdown formatting with headings, code blocks, links, etc.",
        ResponseFormat.YAML_TEXT: "Structure responses in YAML format for machine-readable output.",
        ResponseFormat.JSON_TEXT: "Format responses as a JSON structure for API integration.",
        ResponseFormat.PSEUDO_CODE: "Use pseudocode structure for algorithmic or logical explanations.",
        ResponseFormat.CODE_STRUCTURE: "Structure like code with indentation, comments, and logical blocks."
    }
    return format_instructions.get(self.response_format, "Standard-Formatierung.")

get_length_instructions()

Generiere Längen-spezifische Anweisungen

Source code in toolboxv2/mods/isaa/base/Agent/types.py

def get_length_instructions(self) -> str:
    """Generiere Längen-spezifische Anweisungen"""
    length_instructions = {
        TextLength.MINI_CHAT: "Very short, concise answers (1–2 sentences, max 50 words). Chat style.",
        TextLength.CHAT_CONVERSATION: "Moderate conversation length (2–4 sentences, 50–150 words). Natural conversational style.",
        TextLength.TABLE_CONVERSATION: "Structured, tabular presentation with compact explanations (100–250 words).",
        TextLength.DETAILED_INDEPTH: "Comprehensive, detailed explanations (300–800 words) with depth and context.",
        TextLength.PHD_LEVEL: "Academic depth with extensive explanations (800+ words), references, and technical terminology."
    }
    return length_instructions.get(self.text_length, "Standard-Länge.")

LLMReasonerNode

Bases: AsyncNode

Enhanced strategic reasoning core with outline-driven execution, context management, auto-recovery, and intensive variable system integration.

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

@with_progress_tracking
class LLMReasonerNode(AsyncNode):
    """
    Enhanced strategic reasoning core with outline-driven execution,
    context management, auto-recovery, and intensive variable system integration.
    """

    def __init__(self, max_reasoning_loops: int = 24, **kwargs):
        super().__init__(**kwargs)
        self.max_reasoning_loops = max_reasoning_loops
        self.reasoning_context = []
        self.internal_task_stack = []
        self.meta_tools_registry = {}
        self.current_loop_count = 0
        self.current_reasoning_count = 0
        self.agent_instance: FlowAgent = None

        # Enhanced tracking systems
        self.outline = None
        self.current_outline_step = 0
        self.step_completion_tracking = {}
        self.loop_detection_memory = []
        self.context_summary_threshold = 15
        self.max_context_size = 30
        self.performance_metrics = {
                "loop_times": [],
                "progress_loops": 0,
                "total_loops": 0
            }
        self.auto_recovery_attempts = 0
        self.max_auto_recovery = 8
        self.variable_manager = None

        # Anti-loop mechanisms
        self.last_action_signatures = []
        self.step_enforcement_active = True
        self.mandatory_progress_check = True

    async def prep_async(self, shared):
        """Enhanced initialization with variable system integration"""
        # Reset for new execution
        self.reasoning_context = []
        self.internal_task_stack = []
        self.current_loop_count = 0
        self.current_reasoning_count = 0
        self.outline = None
        self.current_outline_step = 0
        self.step_completion_tracking = {}
        self.loop_detection_memory = []
        self.performance_metrics = {
            "loop_times": [],
            "progress_loops": 0,
            "total_loops": 0
        }
        self.auto_recovery_attempts = 0
        self.last_action_signatures = []

        self.agent_instance = shared.get("agent_instance")

        # Enhanced variable manager integration
        self.variable_manager = shared.get("variable_manager", self.agent_instance.variable_manager)
        context_manager = shared.get("context_manager")

        if self.variable_manager:
            # Store reasoning session context
            session_context = {
                "session_id": shared.get("session_id", "default"),
                "start_time": datetime.now().isoformat(),
                "query": shared.get("current_query", ""),
                "reasoning_mode": "outline_driven"
            }
            self.variable_manager.set("reasoning.current_session", session_context)
            # Load previous successful patterns from variables
            self._load_historical_patterns()

        #Build comprehensive system context via UnifiedContextManager
        system_context = await self._build_enhanced_system_context_unified(shared, context_manager)

        return {
            "original_query": shared.get("current_query", ""),
            "session_id": shared.get("session_id", "default"),
            "agent_instance": shared.get("agent_instance"),
            "variable_manager": self.variable_manager,
            "context_manager": context_manager,  #Context Manager Reference
            "system_context": system_context,
            "available_tools": shared.get("available_tools", []),
            "tool_capabilities": shared.get("tool_capabilities", {}),
            "fast_llm_model": shared.get("fast_llm_model"),
            "complex_llm_model": shared.get("complex_llm_model"),
            "progress_tracker": shared.get("progress_tracker"),
            "formatted_context": shared.get("formatted_context", {}),
            "historical_context": await self._get_historical_context_unified(context_manager, shared.get("session_id")),
            "capabilities_summary": shared.get("capabilities_summary", ""),
            # Sub-system references
            "llm_tool_node": shared.get("llm_tool_node_instance"),
            "task_planner": shared.get("task_planner_instance"),
            "task_executor": shared.get("task_executor_instance"),
        }

    async def exec_async(self, prep_res):
        """Enhanced main reasoning loop with outline-driven execution"""
        if not LITELLM_AVAILABLE:
            return await self._fallback_direct_response(prep_res)

        original_query = prep_res["original_query"]
        agent_instance = prep_res["agent_instance"]
        progress_tracker = prep_res.get("progress_tracker")

        # Initialize enhanced reasoning context
        await self._initialize_reasoning_session(prep_res, original_query)

        # STEP 1: MANDATORY OUTLINE CREATION
        if not self.outline:
            with Spinner("Creating initial outline..."):
                outline_result = await self._create_initial_outline(prep_res)
            if self.outline and len(self.outline.get("steps", [])) == 1:
                # fast llm respose on the input metoning tis is a direct respose and evalute if the input dosent need an outline
                print("Fast direct response triggered")
                response = await self.agent_instance.a_run_llm_completion(
                    model=prep_res.get("fast_llm_model", "openrouter/anthropic/claude-3-haiku"),
                    messages=[{"role": "user", "content": prep_res["original_query"]}],
                    temperature=0.3,
                    max_tokens=2048,
                    node_name="LLMReasonerNode",
                    task_id="fast_direct_response"
                )
                return {
                        "final_result": response,
                        "reasoning_loops": self.current_loop_count,
                        "reasoning_context": self.reasoning_context.copy(),
                        "internal_task_stack": self.internal_task_stack.copy(),
                        "outline": self.outline,
                        "outline_completion": self.current_outline_step,
                        "performance_metrics": self.performance_metrics,
                        "auto_recovery_attempts": self.auto_recovery_attempts
                    }
            elif not outline_result:
                return await self._fallback_direct_response(prep_res)

        final_result = None
        consecutive_no_progress = 0
        max_no_progress = 3

        # Enhanced main reasoning loop with strict progress tracking
        while self.current_reasoning_count < self.max_reasoning_loops:
            self.current_loop_count += 1
            loop_start_time = time.time()

            # Check for infinite loops
            if self._detect_infinite_loop():
                await self._trigger_auto_recovery(prep_res)
                if self.auto_recovery_attempts >= self.max_auto_recovery:
                    break

            # Auto-context management
            await self._manage_context_size()

            # Progress tracking
            if progress_tracker:
                await progress_tracker.emit_event(ProgressEvent(
                    event_type="reasoning_loop",
                    timestamp=time.time(),
                    node_name="LLMReasonerNode",
                    status=NodeStatus.RUNNING,
                    metadata={
                        "loop_number": self.current_loop_count,
                        "outline_step": self.current_outline_step,
                        "outline_total": len(self.outline.get("steps", [])) if self.outline else 0,
                        "context_size": len(self.reasoning_context),
                        "task_stack_size": len(self.internal_task_stack),
                        "auto_recovery_attempts": self.auto_recovery_attempts,
                        "performance_metrics": self.performance_metrics
                    }
                ))

            try:
                # Build enhanced reasoning prompt with outline context
                reasoning_prompt = await self._build_outline_driven_prompt(prep_res)

                # Force progress check if needed
                if self.mandatory_progress_check and consecutive_no_progress >= 2:
                    reasoning_prompt += "\n\n**MANDATORY**: You must either complete current outline step or move to next step. No more analysis without action!"

                # LLM reasoning call
                model_to_use = prep_res.get("complex_llm_model", "openrouter/openai/gpt-4o")

                llm_response = await agent_instance.a_run_llm_completion(
                    model=model_to_use,
                    messages=[{"role": "user", "content": reasoning_prompt}],
                    temperature=0.2,  # Lower temperature for more focused execution
                    # max_tokens=3072,
                    node_name="LLMReasonerNode",
                    stop="<immediate_context>",
                    task_id=f"reasoning_loop_{self.current_loop_count}_step_{self.current_outline_step}"
                )

                # Add LLM response to context
                self.reasoning_context.append({
                    "type": "reasoning",
                    "content": llm_response,
                    "loop": self.current_loop_count,
                    "outline_step": self.current_outline_step,
                    "timestamp": datetime.now().isoformat()
                })

                # Parse and execute meta-tool calls with enhanced tracking
                progress_made = await self._parse_and_execute_meta_tools(llm_response, prep_res)

                action_taken = progress_made.get("action_taken", False)
                actual_progress = progress_made.get("progress_made", False)

                # Update performance with correct progress indication
                self._update_performance_metrics(loop_start_time, actual_progress)

                if not action_taken:
                    self.current_reasoning_count += 1
                    if self.current_outline_step > len(self.outline.get("steps", [])):
                        progress_made["final_result"] = llm_response
                        rprint("Final result reached forced by outline step count")
                    if self.current_outline_step < len(self.outline.get("steps", [])) and self.outline.get("steps", [])[self.current_outline_step].get("is_final", False):
                        progress_made["final_result"] = llm_response
                        rprint("Final result reached forced by outline step count final step")
                else:
                    self.current_reasoning_count -= 1

                # Check for final result
                if progress_made.get("final_result"):
                    final_result = progress_made["final_result"]
                    await self._finalize_reasoning_session(prep_res, final_result)
                    break

                # Progress monitoring
                if progress_made.get("action_taken"):
                    consecutive_no_progress = 0
                    self._update_performance_metrics(loop_start_time, True)
                else:
                    consecutive_no_progress += 1
                    self._update_performance_metrics(loop_start_time, False)

                # Check outline completion
                if self.outline and self.current_outline_step >= len(self.outline.get("steps", []))+self.max_reasoning_loops:
                    # All outline steps completed, force final response
                    final_result = await self._create_outline_completion_response(prep_res)
                    break

                # Emergency break for excessive no-progress
                if consecutive_no_progress >= max_no_progress:
                    await self._trigger_auto_recovery(prep_res)

            except Exception as e:
                await self._handle_reasoning_error(e, prep_res, progress_tracker)
                import traceback
                print(traceback.format_exc())
                if self.auto_recovery_attempts >= self.max_auto_recovery:
                    final_result = await self._create_error_response(original_query, str(e))
                    break


        # If no final result after max loops, create a comprehensive summary
        if not final_result:
            final_result = await self._create_enhanced_timeout_response(original_query, prep_res)

        return {
            "final_result": final_result,
            "reasoning_loops": self.current_loop_count,
            "reasoning_context": self.reasoning_context.copy(),
            "internal_task_stack": self.internal_task_stack.copy(),
            "outline": self.outline,
            "outline_completion": self.current_outline_step,
            "performance_metrics": self.performance_metrics,
            "auto_recovery_attempts": self.auto_recovery_attempts
        }

    async def _build_enhanced_system_context_unified(self, shared, context_manager) -> str:
        """Build comprehensive system context mit UnifiedContextManager"""
        context_parts = []

        # Enhanced agent capabilities
        available_tools = shared.get("available_tools", [])
        if available_tools:
            context_parts.append(f"Available external tools: {', '.join(available_tools)}")

        #Context Manager Status
        if context_manager:
            session_stats = context_manager.get_session_statistics()
            context_parts.append(f"Context System: Advanced with {session_stats['total_sessions']} active sessions")
            context_parts.append(f"Cache Status: {session_stats['cache_entries']} cached contexts")

        # Variable system context
        if self.variable_manager:
            var_info = self.variable_manager.get_scope_info()
            context_parts.append(f"Variable System: {len(var_info)} scopes available")

            # Recent results availability
            results_count = len(self.variable_manager.get("results", {}))
            if results_count:
                context_parts.append(f"Previous results: {results_count} task results available")

        #Enhanced system state mit Context-Awareness
        session_id = shared.get("session_id", "default")
        if context_manager and session_id in context_manager.session_managers:
            session = context_manager.session_managers[session_id]
            if hasattr(session, 'history'):
                context_parts.append(f"Session History: {len(session.history)} conversation entries available")
            elif isinstance(session, dict) and 'history' in session:
                context_parts.append(f"Session History: {len(session['history'])} conversation entries (fallback mode)")

        # System state with enhanced details
        tasks = shared.get("tasks", {})
        if tasks:
            active_tasks = len([t for t in tasks.values() if t.status == "running"])
            completed_tasks = len([t for t in tasks.values() if t.status == "completed"])
            context_parts.append(f"Execution state: {active_tasks} active, {completed_tasks} completed tasks")

        # Performance history
        if hasattr(self, 'historical_successful_patterns'):
            context_parts.append(
                f"Historical patterns: {len(self.historical_successful_patterns)} successful patterns loaded")

        return "\n".join(context_parts) if context_parts else "Basic system context available"

    async def _get_historical_context_unified(self, context_manager, session_id: str) -> str:
        """Get historical context from UnifiedContextManager"""
        if not context_manager:
            return ""

        try:
            #Get unified context for historical analysis
            unified_context = await context_manager.build_unified_context(session_id, None, "historical")

            context_parts = []

            # Chat history insights
            chat_history = unified_context.get("chat_history", [])
            if chat_history:
                context_parts.append(f"Conversation History: {len(chat_history)} messages available")

                # Analyze conversation patterns
                user_queries = [msg['content'] for msg in chat_history if msg.get('role') == 'user']
                if user_queries:
                    avg_query_length = sum(len(q) for q in user_queries) / len(user_queries)
                    context_parts.append(f"Query patterns: Avg length {avg_query_length:.0f} chars")

            # Execution history from variables
            if self.variable_manager:
                # Recent successful queries
                recent_successes = self.variable_manager.get("reasoning.recent_successes", [])
                if recent_successes:
                    context_parts.append(f"Recent successful queries: {len(recent_successes)}")

                # Performance history
                avg_loops = self.variable_manager.get("reasoning.performance.avg_loops", 0)
                if avg_loops:
                    context_parts.append(f"Average reasoning loops: {avg_loops}")

            # System insights from unified context
            execution_state = unified_context.get("execution_state", {})
            if execution_state.get("recent_completions"):
                completions = execution_state["recent_completions"]
                context_parts.append(f"Recent completions: {len(completions)} tasks finished")

            return "\n".join(context_parts)

        except Exception as e:
            eprint(f"Failed to get historical context: {e}")
            return "Historical context unavailable"

    def _load_historical_patterns(self):
        """Load successful patterns from previous reasoning sessions"""
        if not self.variable_manager:
            return

        # Load successful outline patterns
        successful_outlines = self.variable_manager.get("reasoning.successful_patterns.outlines", [])
        failed_patterns = self.variable_manager.get("reasoning.failed_patterns", [])

        self.historical_successful_patterns = successful_outlines[-5:]  # Last 5 successful
        self.historical_failed_patterns = failed_patterns[-10:]  # Last 10 failed

    def _get_historical_context(self) -> str:
        """Get historical context from variable system"""
        if not self.variable_manager:
            return ""

        context_parts = []

        # Recent successful queries
        recent_successes = self.variable_manager.get("reasoning.recent_successes", [])
        if recent_successes:
            context_parts.append(f"Recent successful queries: {len(recent_successes)}")

        # Performance history
        avg_loops = self.variable_manager.get("reasoning.performance.avg_loops", 0)
        if avg_loops:
            context_parts.append(f"Average reasoning loops: {avg_loops}")

        # Common failure patterns to avoid
        failure_patterns = self.variable_manager.get("reasoning.failure_patterns", [])
        if failure_patterns:
            context_parts.append(f"Known failure patterns: {len(failure_patterns)}")

        return "\n".join(context_parts)

    async def _initialize_reasoning_session(self, prep_res, original_query):
        """Initialize enhanced reasoning session with variable tracking"""
        # Initialize reasoning context
        self.reasoning_context.append({
            "type": "session_start",
            "content": f"Enhanced reasoning session started for: {original_query}",
            "timestamp": datetime.now().isoformat(),
            "session_id": prep_res.get("session_id")
        })

        # Store session in variables
        if self.variable_manager:
            session_data = {
                "query": original_query,
                "start_time": datetime.now().isoformat(),
                "max_loops": self.max_reasoning_loops,
                "context_management": "auto_summary",
                "outline_driven": True
            }
            self.variable_manager.set("reasoning.current_session.data", session_data)

        # Add enhanced system context
        self.reasoning_context.append({
            "type": "system_context",
            "content": prep_res["system_context"],
            "timestamp": datetime.now().isoformat()
        })

        # Add historical context if available
        historical = prep_res.get("historical_context")
        if historical:
            self.reasoning_context.append({
                "type": "historical_context",
                "content": historical,
                "timestamp": datetime.now().isoformat()
            })

    async def _create_initial_outline(self, prep_res) -> bool:
        """Create mandatory initial outline, with a fast path for simple queries."""
        original_query = prep_res["original_query"]
        agent_instance = prep_res["agent_instance"]

        outline_prompt = f"""You MUST create an initial execution outline for this query. This is mandatory.

**Query:** {original_query}

**Available Resources:**
- Tools: {', '.join(prep_res.get('available_tools', []))}
- Sub-systems: LLM Tool Node, Task Planner, Task Executor

LLM Tool Node is for all tool calls!
LLM Tool Node is best for simple multi-step tasks like fetching data from a tool and summarizing it.
Task Planner is best for complex tasks with multiple dependencies and complex task flows.

**Historical Context:** {prep_res.get('historical_context', 'None')}

**Fast Path for Simple Queries:**
If the query is simple and can be answered directly without needing tools or complex reasoning, you MUST create a single-step outline using the `direct_response` method.

Create a structured outline using this EXACT format:

```outline
OUTLINE_START
Step 1: [Brief description of first step]
- Method: [internal_reasoning | delegate_to_llm_tool_node | direct_response]
- Expected outcome: [What this step should achieve]
- Success criteria: [How to know this step is complete]

[For complex queries, continue with more steps as needed.]

Final Step: Synthesize results and provide comprehensive response
- Method: direct_response
- Expected outcome: Complete answer to user query
- Success criteria: User query fully addressed
OUTLINE_END
```

**Requirements:**
1. Outline must have between 1 and 7 steps.
2. For simple queries, a single "Final Step" using the 'direct_response' method is the correct approach.
3. Each step must have clear success criteria and build logically toward the answer.
4. Be specific about which meta-tools to use for each step. meta-tools ar not Tools ! avalabel meta-tools *Method* (internal_reasoning, delegate_to_llm_tool_node, direct_response) no exceptions

Create the outline now:"""

        try:
            llm_response = await agent_instance.a_run_llm_completion(
                model=prep_res.get("complex_llm_model", "openrouter/openai/gpt-4o"),
                messages=[{"role": "system", "content": outline_prompt}, {"role": "user", "content": original_query}],
                temperature=0.2,  # Lower temperature for more deterministic outlining
                node_name="LLMReasonerNode",
                task_id="create_initial_outline",
                stream=False,
                auto_fallbacks=False,
                stop=["OUTLINE_END"]
            )
            llm_response += "OUTLINE_END"

            # Parse outline from response
            # print(llm_response)
            outline = self._parse_outline_from_response(llm_response)

            if self.agent_instance and self.agent_instance.progress_tracker:
                await self.agent_instance.progress_tracker.emit_event(ProgressEvent(
                    event_type="outline_created",
                    timestamp=time.time(),
                    node_name="LLMReasonerNode",
                    status=NodeStatus.COMPLETED,
                    task_id="create_initial_outline",
                    metadata={"outline": outline}
                ))

            if outline:
                self.outline = outline
                self.current_outline_step = 0

                # Store outline in variables
                if self.variable_manager:
                    self.variable_manager.set("reasoning.current_session.outline", outline)

                # Add to reasoning context
                self.reasoning_context.append({
                    "type": "outline_created",
                    "content": f"Created outline with {len(outline.get('steps', []))} steps",
                    "outline": outline,
                    "timestamp": datetime.now().isoformat()
                })

                return True
            else:
                return False

        except Exception as e:
            eprint(f"Failed to create initial outline: {e}")
            import traceback
            print(traceback.format_exc())
            return False

    def _parse_outline_from_response(self, response: str) -> dict[str, Any]:
        """Parse structured outline from LLM response"""
        import re

        # Find outline section
        outline_match = re.search(r'OUTLINE_START(.*?)OUTLINE_END', response, re.DOTALL)
        if not outline_match:
            return None

        outline_text = outline_match.group(1).strip()

        # Parse steps
        steps = []
        current_step = None

        for line in outline_text.split('\n'):
            line = line.strip()
            if not line:
                continue

            # New step
            if re.match(r'^Step \d+:', line):
                if current_step:
                    steps.append(current_step)

                current_step = {
                    "description": re.sub(r'^Step \d+:\s*', '', line),
                    "method": "",
                    "expected_outcome": "",
                    "success_criteria": "",
                    "status": "pending"
                }
            elif re.match(r'^Final Step:', line):
                if current_step:
                    steps.append(current_step)

                current_step = {
                    "description": re.sub(r'^Final Step:\s*', '', line),
                    "method": "direct_response",
                    "expected_outcome": "",
                    "success_criteria": "",
                    "status": "pending",
                    "is_final": True
                }
            elif current_step and line.startswith('- Method:'):
                current_step["method"] = line.replace('- Method:', '').strip()
            elif current_step and line.startswith('- Expected outcome:'):
                current_step["expected_outcome"] = line.replace('- Expected outcome:', '').strip()
            elif current_step and line.startswith('- Success criteria:'):
                current_step["success_criteria"] = line.replace('- Success criteria:', '').strip()

        # Add final step if exists
        if current_step:
            steps.append(current_step)

        if not steps:
            return None

        return {
            "steps": steps,
            "created_at": datetime.now().isoformat(),
            "total_steps": len(steps)
        }

    def _build_enhanced_system_context(self, shared) -> str:
        """Build comprehensive system context with variable system info"""
        context_parts = []

        # Enhanced agent capabilities
        available_tools = shared.get("available_tools", [])
        if available_tools:
            context_parts.append(f"Available external tools: {', '.join(available_tools)}")

        # Variable system context
        if self.variable_manager:
            var_info = self.variable_manager.get_scope_info()
            context_parts.append(f"Variable System: {len(var_info)} scopes available")

            # Recent results availability
            results_count = len(self.variable_manager.get("results", {}))
            if results_count:
                context_parts.append(f"Previous results: {results_count} task results available")

        # System state with enhanced details
        tasks = shared.get("tasks", {})
        if tasks:
            active_tasks = len([t for t in tasks.values() if t.status == "running"])
            completed_tasks = len([t for t in tasks.values() if t.status == "completed"])
            context_parts.append(f"Execution state: {active_tasks} active, {completed_tasks} completed tasks")

        # Session context with history
        formatted_context = shared.get("formatted_context", {})
        if formatted_context:
            recent_interaction = formatted_context.get("recent_interaction", "")
            if recent_interaction:
                context_parts.append(f"Recent interaction: {recent_interaction[:100000]}...")

        # Performance history
        if hasattr(self, 'historical_successful_patterns'):
            context_parts.append(
                f"Historical patterns: {len(self.historical_successful_patterns)} successful patterns loaded")

        return "\n".join(context_parts) if context_parts else "Basic system context available"

    async def _manage_context_size(self):
        """Auto-manage context size with intelligent summarization"""
        if len(self.reasoning_context) <= self.context_summary_threshold:
            return

        # Trigger summarization
        if len(self.reasoning_context) >= self.max_context_size:
            # Emergency summarization
            await self._emergency_context_summary()
        elif len(self.reasoning_context) >= self.context_summary_threshold:
            # Regular summarization
            await self._regular_context_summary()

    async def _regular_context_summary(self):
        """Regular context summarization when threshold is reached"""
        # Keep last 10 entries, summarize the rest
        keep_recent = self.reasoning_context[-10:]
        to_summarize = self.reasoning_context[:-10]

        summary = self._create_context_summary(to_summarize, "regular")

        # Replace old context with summary + recent
        self.reasoning_context = [
                                     {
                                         "type": "context_summary",
                                         "content": summary,
                                         "summarized_entries": len(to_summarize),
                                         "summary_type": "regular",
                                         "timestamp": datetime.now().isoformat()
                                     }
                                 ] + keep_recent

    async def _emergency_context_summary(self):
        """Emergency context summarization when max size is reached"""
        # Keep last 5 entries, summarize everything else
        keep_recent = self.reasoning_context[-5:]
        to_summarize = self.reasoning_context[:-5]

        summary = self._create_context_summary(to_summarize, "emergency")

        # Replace with emergency summary
        self.reasoning_context = [
                                     {
                                         "type": "context_summary",
                                         "content": summary,
                                         "summarized_entries": len(to_summarize),
                                         "summary_type": "emergency",
                                         "timestamp": datetime.now().isoformat()
                                     }
                                 ] + keep_recent

    def _create_context_summary(self, entries: list[dict], summary_type: str) -> str:
        """Create intelligent context summary"""
        if not entries:
            return "No context to summarize"

        summary_parts = []

        # Group by type
        by_type = {}
        for entry in entries:
            entry_type = entry.get("type", "unknown")
            if entry_type not in by_type:
                by_type[entry_type] = []
            by_type[entry_type].append(entry)

        # Summarize each type
        for entry_type, type_entries in by_type.items():
            if entry_type == "reasoning":
                reasoning_summary = f"Completed {len(type_entries)} reasoning cycles"
                # Extract key insights
                insights = []
                for entry in type_entries[-3:]:  # Last 3 reasoning entries
                    content = entry.get("content", "")[:1000] + "..."
                    insights.append(content)
                if insights:
                    reasoning_summary += f"\nKey recent reasoning: {'; '.join(insights)}"
                summary_parts.append(reasoning_summary)

            elif entry_type == "meta_tool_result":
                results_summary = f"Executed {len(type_entries)} meta-tool operations"
                # Extract significant results
                significant_results = [
                    entry.get("content", "")[:800]
                    for entry in type_entries
                    if len(entry.get("content", "")) > 50
                ]
                if significant_results:
                    results_summary += f"\nSignificant results: {'; '.join(significant_results[-3:])}"
                summary_parts.append(results_summary)

            else:
                summary_parts.append(f"{entry_type}: {len(type_entries)} entries")

        summary = f"[{summary_type.upper()} SUMMARY] " + "; ".join(summary_parts)

        # Store summary in variables for future reference
        if self.variable_manager:
            summary_data = {
                "type": summary_type,
                "entries_summarized": len(entries),
                "summary": summary,
                "timestamp": datetime.now().isoformat()
            }
            summaries = self.variable_manager.get("reasoning.context_summaries", [])
            summaries.append(summary_data)
            self.variable_manager.set("reasoning.context_summaries", summaries[-10:])  # Keep last 10

        return summary

    def _get_pending_tasks_summary(self) -> str:
        """Get summary of pending tasks requiring attention"""
        if not self.internal_task_stack:
            return "⚠️ NO TASKS IN STACK - You must create tasks from your outline immediately!"

        pending_tasks = [task for task in self.internal_task_stack if task.get("status", "pending") == "pending"]

        if not pending_tasks:
            return "✅ No pending tasks - ready for next outline step or completion"

        task_summaries = []
        for i, task in enumerate(pending_tasks[:3], 1):
            desc = task.get("description", "No description")[:150] + "..." if len(
                task.get("description", "")) > 50 else task.get("description", "")
            step_ref = task.get("outline_step_ref", "")
            step_info = f" ({step_ref})" if step_ref else ""
            task_summaries.append(f"{i}. {desc}{step_info}")

        if len(pending_tasks) > 3:
            task_summaries.append(f"... +{len(pending_tasks) - 3} more pending tasks")

        return f"📋 {len(pending_tasks)} pending tasks:\n" + "\n".join(task_summaries)

    async def _build_outline_driven_prompt(self, prep_res) -> str:
        """Build outline-driven reasoning prompt mit UnifiedContextManager Integration"""

        # Get current task with enhanced visibility
        current_stack_task = self._get_current_stack_task()

        #Enhanced context aus UnifiedContextManager
        context_manager = prep_res.get("context_manager")
        session_id = prep_res.get("session_id", "default")

        # Build unified context sections
        unified_context_summary = ""
        recent_results_context = ""

        if context_manager:
            try:
                # Get full unified context
                unified_context = await  context_manager.build_unified_context(session_id, prep_res.get('original_query'))

                unified_context_summary = self._format_unified_context_for_reasoning(unified_context)
                recent_results_context = self._build_recent_results_from_unified_context(unified_context)
            except Exception as e:
                eprint(f"Failed to get unified context in reasoning prompt: {e}")
                unified_context_summary = "Unified context unavailable"
                recent_results_context = "**No recent results available**"

        # Enhanced context summaries (keeping existing functionality)
        context_summary = self._summarize_reasoning_context()
        task_stack_summary = self._summarize_task_stack()
        outline_status = self._get_current_step_requirements()
        performance_context = self._get_performance_context()

        # Enhanced variable system integration with better suggestions
        variable_context = ""
        variable_suggestions = []
        if self.variable_manager:
            variable_context = self.variable_manager.get_llm_variable_context()
            query_text = prep_res.get('original_query', '')
            if current_stack_task:
                query_text += " " + current_stack_task.get('description', '')
            variable_suggestions = self.variable_manager.get_variable_suggestions(query_text)

        immediate_context = self._get_immediate_context_for_prompt()
        # Detect if we're in a potential loop situation
        loop_warning = self._generate_loop_warning()

        prompt = f"""You are the enhanced strategic reasoning core operating in OUTLINE-DRIVEN MODE with MANDATORY TASK STACK enforcement.
## ABSOLUTE REQUIREMENTS - VIOLATION = IMMEDIATE STOP:
1. **WORK ONLY THROUGH TASK STACK** - No work outside the stack permitted
2. **SEE CURRENT TASK DIRECTLY** - Your current task is shown below
3. **USE VARIABLE SYSTEM** - All results are automatically stored and accessible
4. **USE UNIFIED CONTEXT** - Rich conversation and execution history is available
5. **MARK TASKS COMPLETE** - Every finished task must be marked complete
6. **NO REPEATED ACTIONS** - Check variables first before re-doing work

{loop_warning}

## <CURRENT SITUATION>:
**Original Query:** {prep_res['original_query']}

**Unified Context Summary:**
{unified_context_summary}

**Current Context Summary:**
{context_summary}

**Current Outline Status:**
{outline_status}

** CURRENT TASK FROM STACK:**
{current_stack_task}

**Internal Task Stack:**
{task_stack_summary}

**Performance Metrics:**
{performance_context}

## ENHANCED CONTEXT INTEGRATION:
{variable_context}

** SUGGESTED VARIABLES for current task:**
{', '.join(variable_suggestions[:10]) if variable_suggestions else 'tool_capabilities, query, model_complex, available_tools, timestamp, use_fast_response, tool_registry, name, current_query, current_session'}

** UNIFIED CONTEXT RESULTS ACCESS:**
{recent_results_context}

</CURRENT SITUATION>

## MANDATORY TASK STACK ENFORCEMENT:
**CRITICAL RULE**: You MUST work exclusively through your internal task stack.

**TASK STACK WORKFLOW (MANDATORY):**
1. **CHECK CURRENT TASK**: Your current task is: {current_stack_task.get('description', 'NO CURRENT TASK - ADD TASKS FROM OUTLINE!') if current_stack_task else 'NO CURRENT TASK - VIOLATION!'}

2. **WORK ONLY ON STACK TASKS**: You can ONLY work on tasks that exist in your internal task stack
   - The task you're working on MUST be in the stack with status "pending"
   - Before any action: Verify the task exists in your stack

3. **MANDATORY TASK COMPLETION**: After completing any work, you MUST mark the task as complete
   - Use: META_TOOL_CALL: manage_internal_task_stack(action="complete", task_description="[exact task description]", outline_step_ref="step_X")

4. **CHECK UNIFIED CONTEXT FIRST**: Before any major action, focus your attention to the variable system to see if results already exist
   - Avalabel results are automatically stored in the variable system
   - The unified context above shows available conversation history and execution state

**CURRENT TASK ANALYSIS:**
{self._analyze_current_task(current_stack_task) if current_stack_task else "❌ NO CURRENT TASK - You must add tasks from your outline!"}

## AVAILABLE META-TOOLS:
You have access to these meta-tools to control sub-systems. Use the EXACT syntax shown:
{self.meta_tools_registry if self.meta_tools_registry else ''}

**META_TOOL_CALL: internal_reasoning(thought: str, thought_number: int, total_thoughts: int, next_thought_needed: bool, current_focus: str, key_insights: list[str], potential_issues: list[str], confidence_level: float)**
- Purpose: Structure your thinking process explicitly
- Use for: Any complex analysis, planning, or problem decomposition
- Example: META_TOOL_CALL: internal_reasoning(thought="I need to break this down into steps", thought_number=1, total_thoughts=3, next_thought_needed=true, current_focus="problem analysis", key_insights=["Query requires multiple data sources"], potential_issues=["Data might not be available"], confidence_level=0.8)

**META_TOOL_CALL: manage_internal_task_stack(action: str, task_description: str)**
- Purpose: Manage your high-level to-do list
- Actions: "add", "remove", "complete", "get_current"
- Example: META_TOOL_CALL: manage_internal_task_stack(action="add", task_description="Research competitor analysis data")

**META_TOOL_CALL: delegate_to_llm_tool_node(task_description: str, tools_list: list[str])**
- Purpose: Delegate specific, self-contained tasks requiring external tools
- Use for: Web searches, file operations, API calls, single-, two-, or three-step tool usage
- Example: META_TOOL_CALL: delegate_to_llm_tool_node(task_description="Search for latest news about AI developments", tools_list=["search_web"])
- Rule: always validate delegate_to_llm_tool_node result. will be available in <immediate_context> after execution!

**META_TOOL_CALL: read_from_variables(scope: str, key: str, purpose: str)**
- Unified context data is available in various scopes
- Example: META_TOOL_CALL: read_from_variables(scope="user", key="name", purpose="Gather user information for later reference")

**META_TOOL_CALL: write_to_variables(scope: str, key: str, value: any, description: str)**
- Store important findings immediately
- Example: META_TOOL_CALL: write_to_variables(scope="user", key="name", value="User-Name", description="The users name for later reference")

**META_TOOL_CALL: advance_outline_step(step_completed: bool, completion_evidence: str, next_step_focus: str)**
- Mark outline steps complete when all related tasks done

**META_TOOL_CALL: direct_response(final_answer: str, outline_completion: bool, steps_completed: list[str])**
- ONLY when ALL outline steps complete or no META_TOOL_CALL needed
- final_answer must contain the full final answer for the user with all necessary context and informations ( format in persona style )
- Purpose: End reasoning and provide final answer to user
- Use when: Query is complete or can be answered directly
- Example: META_TOOL_CALL: direct_response(final_answer="Based on my analysis, here are the key findings...")

note: in this interaction only META_TOOL_CALL ar avalabel. for other tools use META_TOOL_CALL: delegate_to_llm_tool_node with the appropriate tool names!

## REASONING STRATEGY:
1. **Start with internal_reasoning** to understand the query and plan approach
2. **Use manage_internal_task_stack** to track high-level steps
3. **Choose the right delegation strategy:**
   - Simple queries → direct_response
   - Up to 3 tool tasks with llm action → delegate_to_llm_tool_node
   - Complex projects → create_and_execute_plan
4. **Monitor progress** and adapt your approach
5. **End with direct_response** when complete

## EXAMPLES OF GOOD REASONING PATTERNS:

**Simple Query Pattern:**
META_TOOL_CALL: internal_reasoning(thought="This is a straightforward question I can answer directly", thought_number=1, total_thoughts=1, next_thought_needed=false, current_focus="direct response", key_insights=["No external data needed"], potential_issues=[], confidence_level=0.9)
META_TOOL_CALL: direct_response(final_answer="...")

**Research Task Pattern:**
META_TOOL_CALL: internal_reasoning(thought="I need to gather information from external sources", ...)
META_TOOL_CALL: manage_internal_task_stack(action="add", task_description="Research topic X")
META_TOOL_CALL: delegate_to_llm_tool_node(task_description="Search for information about X", tools_list=["search_web"])
[Wait for result]
META_TOOL_CALL: internal_reasoning(thought="I have the research data, now I can formulate response", ...)
META_TOOL_CALL: direct_response(final_answer="Based on my research: ...")

**Complex Project Pattern:**
META_TOOL_CALL: internal_reasoning(thought="This requires multiple steps with dependencies", ...)
META_TOOL_CALL: create_and_execute_plan(goals=["Step 1: Gather data A", "Step 2: Gather data B", "Step 3: Analyze A and B together", "Step 4: Create final report"])
[Wait for plan completion]
META_TOOL_CALL: direct_response(final_answer="I've completed your complex request...")

## ENHANCED ANTI-LOOP ENFORCEMENT:
- Current Loop: {self.current_loop_count}/{self.max_reasoning_loops}
- Auto-Recovery Attempts: {getattr(self, 'auto_recovery_attempts', 0)}/{getattr(self, 'max_auto_recovery', 3)}
- Last Actions: {', '.join(getattr(self, 'last_action_signatures', [])[-3:]) if hasattr(self, 'last_action_signatures') else 'None'}

**⚠️ LOOP PREVENTION RULES:**
1. If you just read a variable, DO NOT read the same variable again
2. If you completed a task, DO NOT repeat the same work
3. If results exist in unified context, DO NOT recreate them
4. Always advance to next logical step

{self._get_current_step_requirements()}

## YOUR NEXT ACTION (Choose ONE):
Based on your current task, unified context, and available variables, what is your next concrete action?

**DECISION TREE:**
1. ❓ No current task? → Add tasks from outline
2. 📖 Current task needs data? → Check variables and unified context first (read_from_variables)
3. 🔧 Need to execute tools and reason over up to 3 steps? → Use delegate_to_llm_tool_node
4. ✅ Task complete? → Mark complete and advance
5. 🎯 All outline done? → Provide direct_response

Latest unified context: (note delegation results could be wrong or misleading)
<immediate_context>
{immediate_context}
</immediate_context>

must validate <immediate_context> output!
- validate the <immediate_context> output! before proceeding with the outline!
- output compleat fail -> direct_response
- informations missing or output recovery needed -> repeat step with a different strategy
- not enough structure -> use create_and_execute_plan meta-tool call
- output is valid -> continue with the outline!
- if dynamic Planing is needed, you must use the appropriate meta-tool call

**Remember**:
- work step by step max call 3 meta-tool calls in one run.
- only use direct_response if the outline is complete and context from <immediate_context> is enough to answer the query!
- Your job is to work systematically through your outline using your task stack, while leveraging the unified context system to avoid duplicate work and maintain context."""

        return prompt

    def _format_unified_context_for_reasoning(self, unified_context: dict[str, Any]) -> str:
        """Format unified context für reasoning prompt"""
        try:
            context_parts = []

            # Session info
            session_stats = unified_context.get('session_stats', {})
            context_parts.append(
                f"Session: {unified_context.get('session_id', 'unknown')} with {session_stats.get('current_session_length', 0)} messages")

            # Chat history summary
            chat_history = unified_context.get('chat_history', [])
            if chat_history:
                recent_messages = len([msg for msg in chat_history if msg.get('role') == 'user'])
                context_parts.append(f"Conversation: {recent_messages} user queries in current context")

                # Show last user message for reference
                last_user_msg = None
                for msg in reversed(chat_history):
                    if msg.get('role') == 'user':
                        last_user_msg = msg.get('content', '')[:100] + "..."
                        break
                if last_user_msg:
                    context_parts.append(f"Latest user query: {last_user_msg}")

            # Execution state
            execution_state = unified_context.get('execution_state', {})
            active_tasks = execution_state.get('active_tasks', [])
            recent_completions = execution_state.get('recent_completions', [])
            if active_tasks or recent_completions:
                context_parts.append(
                    f"Execution: {len(active_tasks)} active, {len(recent_completions)} completed tasks")

            # Available data
            variables = unified_context.get('variables', {})
            recent_results = variables.get('recent_results', [])
            if recent_results:
                context_parts.append(f"Available Results: {len(recent_results)} recent task results accessible")

            return "\n".join(context_parts)

        except Exception as e:
            return f"Error formatting unified context: {str(e)}"

    def _build_recent_results_from_unified_context(self, unified_context: dict[str, Any]) -> str:
        """Build recent results context from unified context"""
        try:
            variables = unified_context.get('variables', {})
            recent_results = variables.get('recent_results', [])

            if not recent_results:
                return "**No recent results available from unified context**"

            result_context = """**🔍 RECENT RESULTS FROM UNIFIED CONTEXT:**"""

            for i, result in enumerate(recent_results[:3], 1):  # Top 3 results
                task_id = result.get('task_id', f'result_{i}')
                preview = result.get('preview', 'No preview')
                success = result.get('success', False)
                status_icon = "✅" if success else "❌"

                result_context += f"\n{status_icon} {task_id}: {preview}"

            result_context += "\n\n**Quick Access Keys Available:**"
            result_context += "\n- Use read_from_variables(scope='results', key='task_id.data') for specific results"
            result_context += "\n- Check delegation.latest for most recent delegation results"

            return result_context

        except Exception as e:
            return f"**Error accessing recent results: {str(e)}**"

    def _generate_loop_warning(self) -> str:
        """Generate loop warning if repetitive behavior detected"""
        if len(self.last_action_signatures) >= 3:
            recent_actions = self.last_action_signatures[-3:]
            if len(set(recent_actions)) <= 2:
                return """
⚠️ **LOOP WARNING DETECTED** ⚠️
You are repeating similar actions. MUST change approach:
- If you just read variables, act on the results
- If you delegated tasks, check the results
- Complete current task and advance to next step
- DO NOT repeat the same meta-tool calls
    """
        return ""

    def _get_current_stack_task(self) -> dict[str, Any]:
        """Get current pending task from stack for direct visibility"""
        if not self.internal_task_stack:
            return {}

        pending_tasks = [task for task in self.internal_task_stack if task.get("status", "pending") == "pending"]
        if pending_tasks:
            current_task = pending_tasks[0]  # Get first pending task
            return {
                "description": current_task.get("description", ""),
                "outline_step_ref": current_task.get("outline_step_ref", ""),
                "status": current_task.get("status", "pending"),
                "added_at": current_task.get("added_at", ""),
                "task_index": self.internal_task_stack.index(current_task) + 1,
                "total_tasks": len(self.internal_task_stack)
            }

        return {}

    def _analyze_current_task(self, current_task: dict[str, Any]) -> str:
        """Analyze current task and provide guidance"""
        if not current_task:
            return "❌ NO CURRENT TASK - Add tasks from your outline immediately!"

        description = current_task.get("description", "")
        outline_ref = current_task.get("outline_step_ref", "")

        analysis = f"""CURRENT TASK IDENTIFIED:
Task: {description}
Outline Reference: {outline_ref}
Position: {current_task.get('task_index', '?')}/{current_task.get('total_tasks', '?')}

RECOMMENDED ACTION:"""

        # Analyze task content for recommendations
        if "read" in description.lower() or "file" in description.lower():
            analysis += "\n1. Check if file content already exists in variables (read_from_variables)"
            analysis += "\n2. If not found, use delegate_to_llm_tool_node with read_file tool"
        elif "write" in description.lower() or "create" in description.lower():
            analysis += "\n1. Check if content is ready in variables"
            analysis += "\n2. Use delegate_to_llm_tool_node with write_file tool"
        elif "analyze" in description.lower() or "question" in description.lower():
            analysis += "\n1. Read existing data from variables"
            analysis += "\n2. Process the information and provide direct_response"
        else:
            analysis += "\n1. Break down the task into specific actions"
            analysis += "\n2. Verify last Task Delegation results"

        return analysis

    def _get_immediate_context_for_prompt(self) -> str:
        """Get immediate context additions from recent meta-tool executions"""
        recent_results = [
            entry for entry in self.reasoning_context[-5:]  # Last 5 entries
            if entry.get("type") == "meta_tool_result"
        ]

        if not recent_results:
            return "No recent meta-tool results"

        context_parts = ["📊 IMMEDIATE CONTEXT FROM RECENT ACTIONS:"]

        for result in recent_results:
            meta_tool = result.get("meta_tool", "unknown")
            content = result.get("content", "")
            loop = result.get("loop", "?")

            # Format based on meta-tool type
            if meta_tool == "delegate_to_llm_tool_node":
                context_parts.append(f"✅ DELEGATION RESULT (Loop {loop}):")
                context_parts.append(f"   {content}")
            elif meta_tool == "read_from_variables":
                context_parts.append(f"📖 VARIABLE READ (Loop {loop}):")
                context_parts.append(f"   {content}")
            elif meta_tool == "manage_internal_task_stack":
                context_parts.append(f"📋 TASK UPDATE (Loop {loop}):")
                context_parts.append(f"   {content}")
            else:
                context_parts.append(f"🔧 {meta_tool.upper()} (Loop {loop}):")
                context_parts.append(f"   {content}")

        return "\n".join(context_parts)

    def _summarize_reasoning_context(self) -> str:
        """Enhanced reasoning context summary with immediate result visibility"""
        if not self.reasoning_context:
            return "No previous reasoning steps"

        # Separate different types of context entries
        reasoning_entries = []
        meta_tool_results = []
        errors = []

        for entry in self.reasoning_context:
            entry_type = entry.get("type", "unknown")

            if entry_type == "reasoning":
                reasoning_entries.append(entry)
            elif entry_type == "meta_tool_result":
                meta_tool_results.append(entry)
            elif entry_type == "error":
                errors.append(entry)

        summary_parts = []

        # Show recent meta-tool results FIRST for immediate visibility
        if meta_tool_results:
            summary_parts.append("🔍 RECENT RESULTS:")
            for result in meta_tool_results[-3:]:  # Last 3 results
                meta_tool = result.get("meta_tool", "unknown")
                content = result.get("content", "")[:3000] + "..."
                loop = result.get("loop", "?")
                summary_parts.append(f"  [{meta_tool}] Loop {loop}: {content}")

        # Show reasoning summary
        if reasoning_entries:
            summary_parts.append(f"\n💭 REASONING: {len(reasoning_entries)} reasoning cycles completed")

        # Show errors if any
        if errors:
            summary_parts.append(f"\n⚠️ ERRORS: {len(errors)} errors encountered")
            for error in errors[-2:]:  # Last 2 errors
                content = error.get("content", "")[:1500]
                summary_parts.append(f"  Error: {content}")

        return "\n".join(summary_parts)

    def _get_current_step_requirements(self) -> str:
        """Get requirements for current outline step"""
        if not self.outline or not self.outline.get("steps"):
            return "ERROR: No outline available"

        steps = self.outline["steps"]
        if self.current_outline_step >= len(steps):
            return "All outline steps completed - must provide final response"

        current_step = steps[self.current_outline_step]

        requirements = f"""CURRENT STEP FOCUS:
Description: {current_step.get('description', 'Unknown')}
Required Method: {current_step.get('method', 'Unknown')}
Expected Outcome: {current_step.get('expected_outcome', 'Unknown')}
Success Criteria: {current_step.get('success_criteria', 'Unknown')}
Current Status: {current_step.get('status', 'pending')}

You MUST use the specified method and achieve the expected outcome before advancing."""

        return requirements

    def _get_performance_context(self) -> str:
        """Get performance context with accurate metrics"""
        if not self.performance_metrics:
            return "No performance metrics available"

        metrics_parts = []

        # Core metrics
        avg_time = self.performance_metrics.get("avg_loop_time", 0)
        efficiency = self.performance_metrics.get("action_efficiency", 0)
        total_loops = self.performance_metrics.get("total_loops", 0)
        progress_loops = self.performance_metrics.get("progress_loops", 0)

        metrics_parts.append(f"Avg Loop Time: {avg_time:.2f}s")
        metrics_parts.append(f"Progress Rate: {efficiency:.1%}")
        metrics_parts.append(f"Action Efficiency: {efficiency:.1%}")

        # Performance warnings
        if total_loops > 3 and efficiency < 0.5:
            metrics_parts.append("⚠️ LOW EFFICIENCY - Need more progress actions")
        elif total_loops > 5 and efficiency < 0.3:
            metrics_parts.append("🔴 VERY LOW EFFICIENCY - Review approach")

        # Loop detection warning based on actual metrics
        if len(self.last_action_signatures) > 3:
            unique_recent = len(set(self.last_action_signatures[-3:]))
            if unique_recent <= 1:
                metrics_parts.append("⚠️ LOOP PATTERN DETECTED - Change approach required")

        return "; ".join(metrics_parts)

    def _track_action_type(self, action_type: str, success: bool = True):
        """Track specific action types for detailed performance analysis"""
        if not hasattr(self, 'action_tracking'):
            self.action_tracking = {}

        if action_type not in self.action_tracking:
            self.action_tracking[action_type] = {"total": 0, "successful": 0}

        self.action_tracking[action_type]["total"] += 1
        if success:
            self.action_tracking[action_type]["successful"] += 1

        # Update overall action efficiency based on all action types
        total_actions = sum(stats["total"] for stats in self.action_tracking.values())
        successful_actions = sum(stats["successful"] for stats in self.action_tracking.values())

        if total_actions > 0:
            self.performance_metrics["detailed_action_efficiency"] = successful_actions / total_actions


    def _detect_infinite_loop(self) -> bool:
        """Enhanced infinite loop detection with multiple patterns"""
        if len(self.last_action_signatures) < 3:
            return False

        # 1. Immediate repetition (same action 3+ times)
        recent_actions = self.last_action_signatures[-3:]
        if len(set(recent_actions)) == 1:
            return True

        # 2. Pattern repetition (AB-AB-AB pattern)
        if len(self.last_action_signatures) >= 6:
            pattern1 = self.last_action_signatures[-6:-3]
            pattern2 = self.last_action_signatures[-3:]
            if pattern1 == pattern2:
                return True

        # 3. Variable read loops (multiple reads of same variable)
        variable_reads = [sig for sig in self.last_action_signatures if sig.startswith("read_from_variables")]
        if len(variable_reads) >= 3:
            # Extract variable signatures from recent reads
            recent_var_reads = variable_reads[-3:]
            if len(set(recent_var_reads)) <= 2:  # Repeated variable reads
                return True

        # 4. No outline progress for extended loops
        if self.current_loop_count > 5:
            if not hasattr(self, '_last_step_progress_loop'):
                self._last_step_progress_loop = {}

            last_progress = self._last_step_progress_loop.get(self.current_outline_step, 0)
            if self.current_loop_count - last_progress > 4:  # No step progress for 4+ loops
                return True

        # 5. Same task stack state for multiple loops
        if hasattr(self, '_task_stack_states'):
            stack_signature = hash(
                str([(t.get('status'), t.get('description')[:20]) for t in self.internal_task_stack]))
            if stack_signature in self._task_stack_states:
                repetitions = self._task_stack_states[stack_signature]
                if repetitions >= 4:
                    return True
                self._task_stack_states[stack_signature] = repetitions + 1
            else:
                self._task_stack_states[stack_signature] = 1
        else:
            self._task_stack_states = {}

        return False

    async def _trigger_auto_recovery(self, prep_res):
        """Trigger auto-recovery mechanism"""
        self.auto_recovery_attempts += 1

        # Store failure pattern
        if self.variable_manager:
            failure_data = {
                "timestamp": datetime.now().isoformat(),
                "loop_count": self.current_loop_count,
                "outline_step": self.current_outline_step,
                "last_actions": self.last_action_signatures[-5:],
                "recovery_attempt": self.auto_recovery_attempts
            }
            failures = self.variable_manager.get("reasoning.failure_patterns", [])
            failures.append(failure_data)
            self.variable_manager.set("reasoning.failure_patterns", failures[-20:])  # Keep last 20

        # Recovery strategies
        if self.auto_recovery_attempts == 1:
            # Force outline step advancement
            await self._force_outline_advancement(prep_res)
        elif self.auto_recovery_attempts == 2:
            # Skip current step and move to next
            await self._emergency_step_skip(prep_res)
        else:
            # Final emergency: force completion
            await self._emergency_completion(prep_res)

    async def _force_outline_advancement(self, prep_res):
        """Force advancement to next outline step"""
        if self.outline and self.current_outline_step < len(self.outline["steps"]):
            current_step = self.outline["steps"][self.current_outline_step]
            current_step["status"] = "force_completed"
            current_step["completion_method"] = "auto_recovery"

            self.current_outline_step += 1

            # Add to context
            self.reasoning_context.append({
                "type": "auto_recovery",
                "content": f"Force advanced to step {self.current_outline_step + 1} due to loop detection",
                "recovery_attempt": self.auto_recovery_attempts,
                "timestamp": datetime.now().isoformat()
            })

    async def _emergency_step_skip(self, prep_res):
        """Emergency skip of problematic step"""
        if self.outline and self.current_outline_step < len(self.outline["steps"]) - 1:
            current_step = self.outline["steps"][self.current_outline_step]
            current_step["status"] = "emergency_skipped"
            current_step["skip_reason"] = "loop_recovery"

            self.current_outline_step += 1

            # Add to context
            self.reasoning_context.append({
                "type": "emergency_skip",
                "content": f"Emergency skipped step {self.current_outline_step} and advanced to step {self.current_outline_step + 1}",
                "recovery_attempt": self.auto_recovery_attempts,
                "timestamp": datetime.now().isoformat()
            })

    async def _emergency_completion(self, prep_res):
        """Emergency completion of reasoning"""
        # Mark all remaining steps as emergency completed
        if self.outline:
            for i in range(self.current_outline_step, len(self.outline["steps"])):
                self.outline["steps"][i]["status"] = "emergency_completed"

            self.current_outline_step = len(self.outline["steps"])

        # Add to context
        self.reasoning_context.append({
            "type": "emergency_completion",
            "content": "Emergency completion triggered due to excessive recovery attempts",
            "recovery_attempt": self.auto_recovery_attempts,
            "timestamp": datetime.now().isoformat()
        })

    def _update_performance_metrics(self, loop_start_time: float, progress_made: bool):
        """Update performance metrics with accurate action efficiency tracking"""
        loop_duration = time.time() - loop_start_time

        # Initialize metrics if needed
        if not hasattr(self, 'performance_metrics') or not self.performance_metrics:
            self.performance_metrics = {
                "loop_times": [],
                "progress_loops": 0,
                "total_loops": 0
            }

        # Update core metrics
        self.performance_metrics["loop_times"].append(loop_duration)
        self.performance_metrics["total_loops"] += 1

        if progress_made:
            self.performance_metrics["progress_loops"] += 1

        # Calculate derived metrics
        total = self.performance_metrics["total_loops"]
        progress = self.performance_metrics["progress_loops"]

        self.performance_metrics["avg_loop_time"] = sum(self.performance_metrics["loop_times"]) / len(
            self.performance_metrics["loop_times"])
        self.performance_metrics["action_efficiency"] = progress / total if total > 0 else 0.0
        self.performance_metrics["progress_rate"] = self.performance_metrics["action_efficiency"]  # Same metric

        # Keep only recent loop times for memory efficiency
        if len(self.performance_metrics["loop_times"]) > 10:
            self.performance_metrics["loop_times"] = self.performance_metrics["loop_times"][-10:]

    def _add_context_to_reasoning(self, context_addition: str, meta_tool_name: str,
                                  execution_details: dict = None) -> None:
        """Add context addition to reasoning context for immediate visibility in next LLM prompt"""
        if not context_addition:
            return

        # Create structured context entry
        context_entry = {
            "type": "meta_tool_result",
            "content": context_addition,
            "meta_tool": meta_tool_name,
            "loop": self.current_loop_count,
            "outline_step": getattr(self, 'current_outline_step', 0),
            "timestamp": datetime.now().isoformat()
        }

        # Add execution details if provided
        if execution_details:
            context_entry["execution_details"] = {
                "duration": execution_details.get("execution_duration", 0),
                "success": execution_details.get("execution_success", False),
                "tool_category": execution_details.get("tool_category", "unknown")
            }

        # Add to reasoning context for immediate visibility
        self.reasoning_context.append(context_entry)

        # Store in variables for persistent access
        if self.agent_instance:
            if not self.agent_instance.shared.get("system_context"):
                self.agent_instance.shared["system_context"] = {}
            if not self.agent_instance.shared["system_context"].get("reasoning_context"):
                self.agent_instance.shared["system_context"]["reasoning_context"] = {}

            result_key = f"reasoning.loop_{self.current_loop_count}_{meta_tool_name}"
            self.agent_instance.shared["system_context"]["reasoning_context"][result_key] = {
                "context_addition": context_addition,
                "meta_tool": meta_tool_name,
                "timestamp": datetime.now().isoformat(),
                "loop": self.current_loop_count
            }

    async def _parse_and_execute_meta_tools(self, llm_response: str, prep_res: dict) -> dict[str, Any]:
        """Enhanced meta-tool parsing with comprehensive progress tracking"""

        result = {
            "final_result": None,
            "action_taken": None,
            "progress_made": False,
            "context_addition": None
        }

        progress_tracker = prep_res.get("progress_tracker")
        session_id = prep_res.get("session_id")

        # Pattern to match META_TOOL_CALL: tool_name(args...)
        pattern = r'META_TOOL_CALL:'
        matches = _extract_meta_tool_calls(llm_response, pattern)

        if not matches and progress_tracker:
            # No meta-tools found in response
            await progress_tracker.emit_event(ProgressEvent(
                event_type="meta_tool_analysis",
                node_name="LLMReasonerNode",
                session_id=session_id,
                status=NodeStatus.COMPLETED,
                success=True,  # Die Analyse selbst war erfolgreich
                node_phase="analysis_complete",  # Verwendung des dedizierten Feldes
                llm_output=llm_response,  # Speichert die vollständige analysierte Antwort
                metadata={
                    "analysis_result": "no_meta_tools_detected",
                    "reasoning_loop": self.current_loop_count,
                    "outline_step": self.current_outline_step if hasattr(self, 'current_outline_step') else 0,
                    "context_size": len(self.reasoning_context),
                    "performance_warning": len(self.reasoning_context) > 10 and self.current_loop_count > 5
                }
            ))
            result["context_addition"] = "No action taken - this violates outline-driven execution requirements"
            self._add_context_to_reasoning(result["context_addition"], "invalid", {})

            return result

        for i, (tool_name, args_str) in enumerate(matches):
            meta_tool_start = time.perf_counter()

            # Track action signature for loop detection
            action_signature = f"{tool_name}:{hash(args_str) % 1000}"
            self.last_action_signatures.append(action_signature)
            if len(self.last_action_signatures) > 10:
                self.last_action_signatures = self.last_action_signatures[-10:]

            try:
                # Parse arguments with enhanced error handling
                args = _parse_tool_args(args_str)
                if progress_tracker:
                    await progress_tracker.emit_event(ProgressEvent(
                        event_type="tool_call",  # Vereinheitlicht auf "tool_call"
                        node_name="LLMReasonerNode",
                        session_id=session_id,
                        status=NodeStatus.RUNNING,
                        tool_name=tool_name,
                        is_meta_tool=True,  # Klares Flag für Meta-Tools
                        tool_args=args,
                        task_id=f"meta_tool_{tool_name}_{i + 1}",
                        metadata={
                            "reasoning_loop": self.current_loop_count,
                            "outline_step": self.current_outline_step if hasattr(self, 'current_outline_step') else 0
                        }
                    ))
                rprint(f"Parsed args: {args}")

                # Execute meta-tool with detailed tracking
                meta_result = None
                execution_details = {
                    "meta_tool_name": tool_name,
                    "parsed_args": args,
                    "execution_success": False,
                    "execution_duration": 0.0,
                    "reasoning_loop": self.current_loop_count,
                    "outline_step": self.current_outline_step if hasattr(self, 'current_outline_step') else 0,
                    "context_before_size": len(self.reasoning_context),
                    "task_stack_before_size": len(self.internal_task_stack),
                    "tool_category": self._get_tool_category(tool_name),
                    "execution_phase": "executing"
                }

                if tool_name == "internal_reasoning":
                    meta_result = await self._execute_enhanced_internal_reasoning(args, prep_res)
                    execution_details.update({
                        "thought_number": args.get("thought_number", 1),
                        "total_thoughts": args.get("total_thoughts", 1),
                        "current_focus": args.get("current_focus", ""),
                        "confidence_level": args.get("confidence_level", 0.5),
                        "key_insights": args.get("key_insights", []),
                        "key_insights_count": len(args.get("key_insights", [])),
                        "potential_issues_count": len(args.get("potential_issues", [])),
                        "next_thought_needed": args.get("next_thought_needed", False),
                        "internal_reasoning_log_size": len(getattr(self, 'internal_reasoning_log', [])),
                        "reasoning_depth": self._calculate_reasoning_depth(),
                        "outline_step_progress": args.get("outline_step_progress", "")
                    })
                    result["action_taken"] = False

                elif tool_name == "manage_internal_task_stack":
                    meta_result = await self._execute_enhanced_task_stack(args, prep_res)
                    execution_details.update({
                        "stack_action": args.get("action", "unknown"),
                        "task_description": args.get("task_description", ""),
                        "outline_step_ref": args.get("outline_step_ref", ""),
                        "stack_size_before": len(self.internal_task_stack),
                        "stack_size_after": 0  # Will be updated below
                    })
                    execution_details["stack_size_after"] = len(self.internal_task_stack)
                    execution_details["stack_change"] = execution_details["stack_size_after"] - execution_details[
                        "stack_size_before"]
                    result["action_taken"] = True

                elif tool_name == "delegate_to_llm_tool_node":
                    meta_result = await self._execute_enhanced_delegate_llm_tool(args, prep_res)
                    execution_details.update({
                        "delegated_task_description": args.get("task_description", ""),
                        "tools_list": args.get("tools_list", []),
                        "tools_count": len(args.get("tools_list", [])),
                        "delegation_target": "LLMToolNode",
                        "sub_system_execution": True,
                        "delegation_complexity": self._assess_delegation_complexity(args),
                        "outline_step_completion": args.get("outline_step_completion", False)
                    })
                    result["action_taken"] = True
                    result["progress_made"] = True

                elif False and tool_name == "create_and_execute_plan":
                    meta_result = await self._execute_enhanced_create_plan(args, prep_res)
                    execution_details.update({
                        "goals_list": args.get("goals", []),
                        "goals_count": len(args.get("goals", [])),
                        "plan_execution_target": "TaskPlanner_TaskExecutor",
                        "sub_system_execution": True,
                        "complex_workflow": True,
                        "estimated_complexity": self._estimate_plan_complexity(args.get("goals", [])),
                        "outline_step_completion": args.get("outline_step_completion", False)
                    })
                    result["action_taken"] = True
                    result["progress_made"] = True

                elif False and tool_name == "create_and_run_micro_plan":
                    meta_result = await self._execute_create_and_run_micro_plan(args, prep_res)
                    execution_details.update({
                        "plan_data": args.get("plan_data", {}),
                        "plan_tasks_count": len(args.get("plan_data", {}).get("tasks", [])),
                        "sub_system_execution": True,
                        "delegation_target": "TaskExecutorNode"
                    })
                    result["action_taken"] = True
                    result["progress_made"] = True

                elif tool_name == "advance_outline_step":
                    meta_result = await self._execute_advance_outline_step(args, prep_res)
                    execution_details.update({
                        "step_completed": args.get("step_completed", False),
                        "completion_evidence": args.get("completion_evidence", ""),
                        "next_step_focus": args.get("next_step_focus", ""),
                        "outline_advancement": True,
                        "step_progression": f"{self.current_outline_step}/{len(self.outline.get('steps', [])) if self.outline else 0}"
                    })
                    result["action_taken"] = True
                    result["progress_made"] = True

                elif tool_name == "write_to_variables":
                    meta_result = await self._execute_write_to_variables(args)
                    execution_details.update({
                        "variable_scope": args.get("scope", "reasoning"),
                        "variable_key": args.get("key", ""),
                        "variable_description": args.get("description", ""),
                        "data_persistence": True,
                        "variable_system_operation": "write"
                    })
                    result["action_taken"] = True

                elif tool_name == "read_from_variables":
                    meta_result = await self._execute_read_from_variables(args)
                    execution_details.update({
                        "variable_scope": args.get("scope", "reasoning"),
                        "variable_key": args.get("key", ""),
                        "read_purpose": args.get("purpose", ""),
                        "variable_system_operation": "read",
                        "data_retrieval": True
                    })
                    result["action_taken"] = True

                elif tool_name == "direct_response":

                    final_answer = args.get("final_answer", "Task completed.").replace('\\n', '\n').replace('\\t', '\t')
                    execution_details.update({
                        "final_answer": final_answer,
                        "final_answer_length": len(final_answer),
                        "reasoning_complete": True,
                        "flow_termination": True,
                        "reasoning_summary": self._create_reasoning_summary(),
                        "total_reasoning_steps": len(self.reasoning_context),
                        "outline_completion": True,
                        "steps_completed": args.get("steps_completed", []),
                        "session_completion": True
                    })

                    completion_context = f"✅ REASONING COMPLETE: {final_answer}"
                    self._add_context_to_reasoning(completion_context, tool_name, execution_details)

                    # Store successful completion
                    await self._store_successful_completion(prep_res, final_answer)

                    if progress_tracker:
                        meta_tool_duration = time.perf_counter() - meta_tool_start
                        execution_details["execution_duration"] = meta_tool_duration
                        execution_details["execution_success"] = True

                        await progress_tracker.emit_event(ProgressEvent(
                            event_type="meta_tool_call",
                            timestamp=time.time(),
                            node_name="LLMReasonerNode",
                            status=NodeStatus.COMPLETED,
                            session_id=session_id,
                            task_id=f"meta_tool_{tool_name}_{i + 1}",
                            node_duration=meta_tool_duration,
                            success=True,
                            metadata=execution_details
                        ))

                    result["final_result"] = final_answer
                    result["action_taken"] = True
                    result["progress_made"] = True
                    return result

                # test if tool name is meta_tools_registry if so try to run it
                elif tool_name in self.meta_tools_registry:
                    function = self.meta_tools_registry[tool_name]
                    meta_result = await function(**args)
                    result["action_taken"] = True
                    result["progress_made"] = True
                    execution_details.update({
                        "tool_name": tool_name,
                        "tool_args": args,
                        "tool_result": meta_result
                    })

                # test if tool name is in agent tools if so try to run it
                elif tool_name in self.agent_instance.tool_registry:
                    meta_result = await self.agent_instance.arun_function(tool_name, **args)
                    result["action_taken"] = True
                    result["progress_made"] = True
                    execution_details.update({
                        "tool_name": tool_name,
                        "tool_args": args,
                        "tool_result": meta_result
                    })

                else:
                    execution_details.update({
                        "error_type": "unknown_meta_tool",
                        "error_message": f"Unknown meta-tool: {tool_name}",
                        "execution_success": False,
                        "available_meta_tools": ["internal_reasoning", "manage_internal_task_stack",
                                            "delegate_to_llm_tool_node", "create_and_execute_plan",
                                            "advance_outline_step", "write_to_variables", "read_from_variables",
                                            "direct_response"]
                    })

                    if progress_tracker:
                        meta_tool_duration = time.perf_counter() - meta_tool_start
                        await progress_tracker.emit_event(ProgressEvent(
                            event_type="meta_tool_call",
                            timestamp=time.time(),
                            node_name="LLMReasonerNode",
                            status=NodeStatus.FAILED,
                            session_id=session_id,
                            task_id=f"meta_tool_{tool_name}_{i + 1}",
                            node_duration=meta_tool_duration,
                            success=False,
                            metadata=execution_details
                        ))

                    error_context = f"❌ Unknown meta-tool: {tool_name}"
                    self._add_context_to_reasoning(error_context, tool_name, execution_details)
                    wprint(f"Unknown meta-tool: {tool_name}")
                    continue

                # Update execution details with results
                meta_tool_duration = time.perf_counter() - meta_tool_start
                execution_details.update({
                    "execution_duration": meta_tool_duration,
                    "execution_success": True,
                    "context_after_size": len(self.reasoning_context),
                    "task_stack_after_size": len(self.internal_task_stack),
                    "performance_score": self._calculate_tool_performance_score(meta_tool_duration, tool_name),
                    "execution_phase": "completed"
                })
                self._track_action_type(tool_name, success=True)

                # Add result to context
                if meta_result and meta_result.get("context_addition"):
                    result["context_addition"] = meta_result["context_addition"]
                    execution_details["context_addition_length"] = len(meta_result["context_addition"])

                    self._add_context_to_reasoning(meta_result["context_addition"], tool_name, execution_details)

                # Emit success event
                if progress_tracker:
                    await progress_tracker.emit_event(ProgressEvent(
                        event_type="meta_tool_call",
                        timestamp=time.time(),
                        node_name="LLMReasonerNode",
                        status=NodeStatus.COMPLETED,
                        session_id=session_id,
                        task_id=f"meta_tool_{tool_name}_{i + 1}",
                        node_duration=meta_tool_duration,
                        success=True,
                        metadata=execution_details
                    ))

            except Exception as e:
                meta_tool_duration = time.perf_counter() - meta_tool_start
                error_details = {
                    "meta_tool_name": tool_name,
                    "execution_success": False,
                    "execution_duration": meta_tool_duration,
                    "error_type": type(e).__name__,
                    "error_message": str(e),
                    "reasoning_loop": self.current_loop_count,
                    "outline_step": self.current_outline_step if hasattr(self, 'current_outline_step') else 0,
                    "parsed_args": args if 'args' in locals() else None,
                    "raw_args_string": args_str,
                    "execution_phase": "meta_tool_error",
                    "context_size_at_error": len(self.reasoning_context),
                    "task_stack_size_at_error": len(self.internal_task_stack),
                    "tool_category": self._get_tool_category(tool_name),
                    "error_context": self._get_error_context(e),
                    "recovery_recommended": self.auto_recovery_attempts < getattr(self, 'max_auto_recovery', 3)
                }

                if progress_tracker:
                    await progress_tracker.emit_event(ProgressEvent(
                        event_type="meta_tool_call",
                        timestamp=time.time(),
                        node_name="LLMReasonerNode",
                        status=NodeStatus.FAILED,
                        session_id=session_id,
                        task_id=f"meta_tool_{tool_name}_{i + 1}",
                        node_duration=meta_tool_duration,
                        success=False,
                        metadata=error_details
                    ))

                eprint(f"Meta-tool execution failed for {tool_name}: {e}")
                result["context_addition"] = f"Error executing {tool_name}: {str(e)}"

                self._add_context_to_reasoning(result["context_addition"], tool_name, execution_details)

        # Final summary event if multiple meta-tools were processed
        if len(matches) > 1 and progress_tracker:
            batch_performance = self._calculate_batch_performance(matches)
            reasoning_progress = self._assess_reasoning_progress()

            await progress_tracker.emit_event(ProgressEvent(
                event_type="meta_tool_batch_complete",
                timestamp=time.time(),
                node_name="LLMReasonerNode",
                status=NodeStatus.COMPLETED,
                session_id=session_id,
                metadata={
                    "total_meta_tools_processed": len(matches),
                    "reasoning_loop": self.current_loop_count,
                    "outline_step": self.current_outline_step if hasattr(self, 'current_outline_step') else 0,
                    "batch_execution_complete": True,
                    "final_context_size": len(self.reasoning_context),
                    "final_task_stack_size": len(self.internal_task_stack),
                    "meta_tools_executed": [match[0] for match in matches],
                    "execution_phase": "meta_tool_batch_summary",
                    "batch_performance": batch_performance,
                    "reasoning_progress": reasoning_progress,
                    "progress_made": result["progress_made"],
                    "action_taken": result["action_taken"],
                    "outline_status": {
                        "current_step": self.current_outline_step if hasattr(self, 'current_outline_step') else 0,
                        "total_steps": len(self.outline.get('steps', [])) if self.outline else 0,
                        "completion_ratio": (
                                self.current_outline_step / len(self.outline.get('steps', [1]))) if self.outline else 0
                    },
                    "performance_summary": {
                        "loop_efficiency": self.performance_metrics.get("action_efficiency", 0) if hasattr(self,
                                                                                                           'performance_metrics') else 0,
                        "recovery_attempts": getattr(self, 'auto_recovery_attempts', 0),
                        "context_management_active": len(self.reasoning_context) >= getattr(self,
                                                                                            'context_summary_threshold',
                                                                                            15)
                    }
                }
            ))

        return result

    async def _execute_enhanced_internal_reasoning(self, args: dict, prep_res: dict) -> dict[str, Any]:
        """Enhanced internal reasoning with outline step tracking"""
        # Standard internal reasoning execution
        result = await self._execute_internal_reasoning(args, prep_res)

        # Enhanced with outline step progress
        outline_step_progress = args.get("outline_step_progress", "")
        if outline_step_progress and result:
            result["context_addition"] += f"\nOutline Step Progress: {outline_step_progress}"

        # Track reasoning depth for current step
        if not hasattr(self, '_step_reasoning_depth'):
            self._step_reasoning_depth = {}

        current_step = self.current_outline_step
        self._step_reasoning_depth[current_step] = self._step_reasoning_depth.get(current_step, 0) + 1

        # Warn if too much reasoning without action
        if self._step_reasoning_depth[current_step] > 3:
            result["context_addition"] += "\n⚠️ WARNING: Too much reasoning without concrete action for current step"

        return result

    async def _execute_enhanced_task_stack(self, args: dict, prep_res: dict) -> dict[str, Any]:
        """Enhanced task stack management with outline step tracking"""
        # Get outline step reference
        outline_step_ref = args.get("outline_step_ref", f"step_{self.current_outline_step}")

        # Execute standard task stack management
        result = await self._execute_manage_task_stack(args, prep_res)

        # Enhanced with outline step reference
        if result:
            result["context_addition"] += f"\n[Linked to: {outline_step_ref}]"

        return result

    async def _execute_enhanced_delegate_llm_tool(self, args: dict, prep_res: dict) -> dict[str, Any]:
        """Enhanced delegation with immediate result visibility and guaranteed storage"""
        task_description = args.get("task_description", "")
        tools_list = args.get("tools_list", [])
        outline_step_completion = args.get("outline_step_completion", False)

        # Generate unique delegation ID for this execution
        delegation_id = f"delegation_loop_{self.current_loop_count}"

        # Prepare shared state for LLMToolNode with enhanced result capture
        llm_tool_shared = {
            "current_task_description": task_description,
            "current_query": task_description,
            "formatted_context": {
                "recent_interaction": f"Reasoner delegating task: {task_description}",
                "session_summary": self._get_reasoning_summary(),
                "task_context": f"Loop {self.current_loop_count} delegation - CAPTURE ALL RESULTS"
            },
            "variable_manager": prep_res.get("variable_manager"),
            "agent_instance": prep_res.get("agent_instance"),
            "available_tools": tools_list,
            "tool_capabilities": prep_res.get("tool_capabilities", {}),
            "fast_llm_model": prep_res.get("fast_llm_model"),
            "complex_llm_model": prep_res.get("complex_llm_model"),
            "progress_tracker": prep_res.get("progress_tracker"),
            "session_id": prep_res.get("session_id"),
            "use_fast_response": True
        }

        try:
            # Execute LLMToolNode
            llm_tool_node = LLMToolNode()
            await llm_tool_node.run_async(llm_tool_shared)

            # IMMEDIATE RESULT EXTRACTION - Critical for visibility
            final_response = llm_tool_shared.get("current_response", "No response captured")
            tool_calls_made = llm_tool_shared.get("tool_calls_made", 0)
            tool_results = llm_tool_shared.get("results", {})

            # GUARANTEED STORAGE - Multiple storage patterns for reliability
            delegation_result = {
                "task_description": task_description,
                "tools_used": tools_list,
                "tool_calls_made": tool_calls_made,
                "final_response": final_response,
                "results": tool_results,
                "timestamp": datetime.now().isoformat(),
                "delegation_id": delegation_id,
                "outline_step": self.current_outline_step,
                "reasoning_loop": self.current_loop_count,
                "success": True
            }

            # CRITICAL: Store immediately with multiple access patterns
            if self.variable_manager:
                # 1. Primary delegation storage
                self.variable_manager.set(f"delegation.loop_{self.current_loop_count}", delegation_result)

                # 2. Latest results quick access
                self.variable_manager.set("delegation.latest", delegation_result)

                # 3. Store individual tool results with direct access
                for result_id, result_data in tool_results.items():
                    self.variable_manager.set(f"results.{result_id}.data", result_data.get("data"))

                # 4. Create smart access keys for common patterns
                if "read_file" in tools_list and tool_results:
                    file_content = next((res.get("data") for res in tool_results.values()
                                         if res.get("data") and isinstance(res.get("data"), str)), None)
                    if file_content:
                        self.variable_manager.set("var.file_content", file_content)
                        self.variable_manager.set("latest_file_content", file_content)

                # 5. Update delegation index for discovery
                index = self.variable_manager.get("delegation.index", [])
                index.append({
                    "loop": self.current_loop_count,
                    "task": task_description[:100],
                    "tools": tools_list,
                    "timestamp": datetime.now().isoformat(),
                    "results_available": len(tool_results) > 0
                })
                self.variable_manager.set("delegation.index", index[-20:])

            # Create comprehensive context addition with IMMEDIATE VISIBILITY
            context_addition = f"""DELEGATION COMPLETED (Loop {self.current_loop_count}):
Task: {task_description}
Tools: {', '.join(tools_list)}
Calls Made: {tool_calls_made}
Results Captured: {len(tool_results)} items

FINAL RESULT: {final_response}

- reference variable: delegation.loop_{self.current_loop_count}
DELEGATION END
"""

            # Mark outline step completion if specified
            if outline_step_completion:
                await self._mark_step_completion(prep_res, "delegation_complete", context_addition)

            return {"context_addition": context_addition}

        except Exception as e:
            error_msg = f"❌ DELEGATION FAILED: {str(e)}"
            # Store error for debugging
            if self.variable_manager:
                error_data = {
                    "task": task_description,
                    "error": str(e),
                    "timestamp": datetime.now().isoformat(),
                    "loop": self.current_loop_count
                }
                self.variable_manager.set(f"delegation.error.loop_{self.current_loop_count}", error_data)

            return {"context_addition": error_msg}

    async def _execute_enhanced_create_plan(self, args: dict, prep_res: dict) -> dict[str, Any]:
        """Enhanced plan creation with outline step completion tracking"""
        # Check if this completes the outline step
        outline_step_completion = args.get("outline_step_completion", False)

        # Execute standard plan creation
        result = await self._execute_create_plan(args, prep_res)

        # Enhanced with step completion tracking
        if outline_step_completion and result:
            await self._mark_step_completion(prep_res, "create_and_execute_plan", result["context_addition"])
            result["context_addition"] += f"\n✓ OUTLINE STEP {self.current_outline_step + 1} COMPLETED"

        return result

    # Fügen Sie dies innerhalb der Klasse LLMReasonerNode hinzu

    async def _execute_create_and_run_micro_plan(self, args: dict, prep_res: dict) -> dict[str, Any]:
        """
        Erstellt einen kleinen, dynamischen TaskPlan aus den LLM-Daten und führt ihn sofort
        mit dem TaskExecutorNode aus.
        """
        plan_data = args.get("plan_data", {})

        # Validierung der Eingabe
        if not isinstance(plan_data, dict) or "tasks" not in plan_data:
            error_msg = "❌ Micro-Plan-Fehler: Ungültiges `plan_data`-Format. Es muss ein Dictionary mit einem 'tasks'-Schlüssel sein."
            eprint(error_msg)
            return {"context_addition": error_msg}

        try:
            # 1. Task-Objekte aus den Rohdaten erstellen
            tasks = []
            for task_data in plan_data.get("tasks", []):
                task_type = task_data.pop("type", "generic")  # 'type' entfernen, da es kein Task-Argument ist
                task_class = {"LLMTask": LLMTask, "ToolTask": ToolTask, "DecisionTask": DecisionTask}.get(task_type,
                                                                                                          Task)
                tasks.append(task_class(**task_data))

            # 2. TaskPlan-Objekt erstellen
            plan = TaskPlan(
                id=f"micro_plan_{str(uuid.uuid4())[:8]}",
                name=plan_data.get("plan_name", "Dynamischer Micro-Plan"),
                description=plan_data.get("description", "Vom LLMReasoner on-the-fly erstellter Plan"),
                tasks=tasks,
                execution_strategy=plan_data.get("execution_strategy", "sequential")
            )

            # 3. Den TaskExecutorNode vorbereiten und ausführen
            task_executor_instance = prep_res.get("task_executor")
            if not task_executor_instance:
                return {"context_addition": "❌ Micro-Plan-Fehler: TaskExecutorNode-Instanz nicht gefunden."}

            # Shared-State für den Executor-Lauf vorbereiten
            executor_shared = {
                "current_plan": plan,
                "tasks": {task.id: task for task in tasks},
                "variable_manager": self.variable_manager,
                "agent_instance": self.agent_instance,
                "progress_tracker": prep_res.get("progress_tracker"),
                "fast_llm_model": prep_res.get("fast_llm_model"),
                "complex_llm_model": prep_res.get("complex_llm_model"),
                "available_tools": prep_res.get("available_tools", []),
            }

            # 4. Ausführungsschleife für den Executor
            max_cycles = 10
            for i in range(max_cycles):
                result_status = await task_executor_instance.run_async(executor_shared)
                if result_status in ["plan_completed", "execution_error", "needs_dynamic_replan"]:
                    break

            # 5. Ergebnisse zusammenfassen für den Reasoner-Kontext
            final_results = executor_shared.get("results", {})
            completed_tasks = [t for t in tasks if t.status == "completed"]
            failed_tasks = [t for t in tasks if t.status == "failed"]

            summary = f"""✅ Micro-Plan ausgeführt:
    - Plan: '{plan.name}'
    - Status: {len(completed_tasks)} erfolgreich, {len(failed_tasks)} fehlgeschlagen.
    - Ergebnisse sind jetzt in den `results`-Variablen verfügbar (z.B. `{{{{ results.{tasks[0].id}.data }}}}`)."""

            return {"context_addition": summary}

        except Exception as e:
            import traceback
            eprint(f"Fehler bei der Ausführung des Micro-Plans: {e}")
            print(traceback.format_exc())
            return {"context_addition": f"❌ Micro-Plan-Fehler: {str(e)}"}

    async def _execute_advance_outline_step(self, args: dict, prep_res: dict) -> dict[str, Any]:
        """Execute outline step advancement"""
        step_completed = args.get("step_completed", False)
        completion_evidence = args.get("completion_evidence", "")
        next_step_focus = args.get("next_step_focus", "")

        if not self.outline or not self.outline.get("steps"):
            return {"context_addition": "Cannot advance: No outline available"}

        steps = self.outline["steps"]

        if self.current_outline_step >= len(steps):
            return {"context_addition": "Cannot advance: Already at final step"}

        if step_completed:
            # Mark current step as completed
            if self.current_outline_step < len(steps):
                current_step = steps[self.current_outline_step]
                current_step["status"] = "completed"
                current_step["completion_evidence"] = completion_evidence
                current_step["completed_at"] = datetime.now().isoformat()

            # Advance to next step
            self.current_outline_step += 1

            # Store advancement in variables
            if self.variable_manager:
                advancement_data = {
                    "step_completed": self.current_outline_step,
                    "completion_evidence": completion_evidence,
                    "next_step_focus": next_step_focus,
                    "timestamp": datetime.now().isoformat()
                }
                self.variable_manager.set(f"reasoning.step_completions.{self.current_outline_step - 1}",
                                          advancement_data)

            context_addition = f"""✓ STEP {self.current_outline_step} COMPLETED
Evidence: {completion_evidence}
Advanced to Step {self.current_outline_step + 1}/{len(steps)}"""

            if next_step_focus:
                context_addition += f"\nNext Step Focus: {next_step_focus}"

            if self.current_outline_step >= len(steps):
                context_addition += "\n🎯 ALL OUTLINE STEPS COMPLETED - Ready for direct_response"

        else:
            context_addition = f"Step {self.current_outline_step + 1} not yet completed - continue working on current step"

        return {"context_addition": context_addition}

    async def _execute_read_from_variables(self, args: dict) -> dict[str, Any]:
        """Enhanced variable reading with intelligent discovery and loop prevention"""
        if not self.variable_manager:
            return {"context_addition": "❌ Variable system not available"}

        scope = args.get("scope", args.get("query", "reasoning"))
        key = args.get("key", "")
        purpose = args.get("purpose", "")

        # CRITICAL: Check for repeated reads - prevent infinite loops
        read_signature = f"{scope}.{key}"
        if not hasattr(self, '_variable_read_history'):
            self._variable_read_history = []

        # Prevent reading same variable multiple times in short succession
        recent_reads = [r for r in self._variable_read_history if r['signature'] == read_signature]
        if len(recent_reads) >= 2:
            self._variable_read_history.append({
                'signature': read_signature,
                'timestamp': time.time(),
                'loop': self.current_loop_count
            })
            return {
                "context_addition": f"⚠️ LOOP PREVENTION: Already read {read_signature} {len(recent_reads)} times. Try different approach or advance to next task."
            }

        # Record this read attempt
        self._variable_read_history.append({
            'signature': read_signature,
            'timestamp': time.time(),
            'loop': self.current_loop_count
        })

        # Clean old read history (keep last 10)
        if len(self._variable_read_history) > 10:
            self._variable_read_history = self._variable_read_history[-10:]

        if not key:
            return {"context_addition": "❌ Cannot read: No key provided"}

        try:
            # Smart key resolution for common patterns
            resolved_key = self._resolve_smart_key(scope, key)

            # Try direct access first
            value = self.variable_manager.get(resolved_key)

            if value is not None:
                # Format value for display
                value_display = self._format_variable_value(value)

                context_addition = f"""{resolved_key}={value_display}
Access: Successfully retrieved from variable system"""

                return {"context_addition": context_addition}

            else:
                # Enhanced discovery when not found
                discovery_result = self._perform_smart_variable_discovery(scope, key, purpose)
                return {"context_addition": discovery_result}

        except Exception as e:
            return {"context_addition": f"❌ Variable read error: {str(e)}"}

    def _resolve_smart_key(self, scope: str, key: str) -> str:
        """Resolve smart key patterns for common access cases"""
        # Handle delegation results specially
        if scope == "delegation" and "loop_" in key:
            return f"delegation.{key}"
        elif scope == "results" and key.endswith(".data"):
            return f"results.{key}"
        elif scope == "var" or key.startswith("var."):
            return key if key.startswith("var.") else f"var.{key}"
        else:
            return f"{scope}.{key}" if scope != "reasoning" else f"reasoning.{key}"

    def _format_variable_value(self, value: any) -> str:
        """Format variable value for display with intelligent truncation"""
        if isinstance(value, dict | list):
            value_str = json.dumps(value, default=str, indent=2)
        else:
            value_str = str(value)

        # Smart truncation based on content type
        if len(value_str) > 200000:
            if isinstance(value, dict) and "results" in str(value):
                # For result dicts, show structure
                return f"RESULTS DICT ({len(value)} keys):\n" + value_str[:150000] + "\n... [TRUNCATED]"
            elif isinstance(value, str) and (value.startswith("# ") or "markdown" in value.lower()):
                # For file content, show beginning
                return f"FILE CONTENT ({len(value_str)} chars):\n" + value_str[:100000] + "\n... [FULL CONTENT AVAILABLE]"
            else:
                return value_str[:100000] + f"\n... [TRUNCATED - {len(value_str)} total chars]"

        return value_str

    def _perform_smart_variable_discovery(self, scope: str, key: str, purpose: str) -> str:
        """Perform intelligent variable discovery when key not found"""
        # Check latest delegation results first
        latest = self.variable_manager.get("delegation.latest")
        if latest:
            discovery_msg = f"❌ Variable not found: {scope}.{key}\n\n✨ LATEST DELEGATION RESULTS AVAILABLE:"
            discovery_msg += f"\nTask: {latest.get('task_description', 'Unknown')[:100]}"
            discovery_msg += f"\nResults: {len(latest.get('results', {}))} items available"
            discovery_msg += "\nAccess with: delegation.latest"

            # Show actual keys available
            if latest.get('results'):
                discovery_msg += "\n\n🔍 Available result keys:"
                for result_id in latest['results']:
                    discovery_msg += f"\n• results.{result_id}.data"

            return discovery_msg

        # Check delegation index for recent activity
        index = self.variable_manager.get("delegation.index", [])
        if index:
            recent = index[-3:]  # Last 3 delegations
            discovery_msg = f"❌ Variable not found: {scope}.{key}\n\n📚 RECENT DELEGATIONS:"
            for entry in recent:
                discovery_msg += f"\n• Loop {entry['loop']}: {entry['task'][:50]}..."
                discovery_msg += f"  Access: delegation.loop_{entry['loop']}"
            return discovery_msg

        # Fallback: show available scopes
        available_vars = self.variable_manager.get_available_variables()
        return f"❌ Variable not found: {scope}.{key}\n\n📋 Available scopes: {', '.join(available_vars.keys())}"


    async def _execute_write_to_variables(self, args: dict) -> dict[str, Any]:
        """Enhanced variable writing with automatic result storage"""
        if not self.variable_manager:
            return {"context_addition": "❌ Variable system not available"}

        scope = args.get("scope", "reasoning")
        key = args.get("key", "")
        value = args.get("value", "")
        description = args.get("description", "")

        if not key:
            return {"context_addition": "❌ Cannot write to variables: No key provided"}

        try:
            # Create scoped key
            full_key = f"{scope}.{key}" if scope != "reasoning" else f"reasoning.{key}"

            # Write to variables
            self.variable_manager.set(full_key, value)

            # Store enhanced metadata
            metadata = {
                "description": description,
                "written_at": datetime.now().isoformat(),
                "outline_step": getattr(self, 'current_outline_step', 0),
                "reasoning_loop": self.current_loop_count,
                "value_type": type(value).__name__,
                "value_size": len(str(value)) if value else 0,
                "auto_stored": False  # Manual storage
            }
            self.variable_manager.set(f"{full_key}_metadata", metadata)

            # Update storage index for easy discovery
            storage_index = self.variable_manager.get("reasoning.storage_index", [])
            storage_entry = {
                "key": full_key,
                "description": description,
                "timestamp": datetime.now().isoformat(),
                "loop": self.current_loop_count
            }
            storage_index.append(storage_entry)
            self.variable_manager.set("reasoning.storage_index", storage_index[-20:])  # Keep last 20

            context_addition = f"✅ Stored in variables: {full_key}"
            if description:
                context_addition += f"\n📄 Description: {description}"

            # Show how to access it
            context_addition += f"\n🔍 Access with: read_from_variables(scope=\"{scope}\", key=\"{key}\", purpose=\"...\")"

            return {"context_addition": context_addition}

        except Exception as e:
            return {"context_addition": f"❌ Failed to write to variables: {str(e)}"}

    def _auto_store_delegation_results(self, delegation_result: dict, task_description: str) -> str:
        """Automatically store delegation results with smart naming and comprehensive indexing"""
        if not self.variable_manager:
            return "\n❌ Variable system not available for auto-storage"

        storage_summary = []

        try:
            # Store main delegation result with loop reference
            main_key = f"delegation.loop_{self.current_loop_count}"
            self.variable_manager.set(main_key, delegation_result)
            storage_summary.append(f"• {main_key}")

            # Store individual tool results with smart naming
            results = delegation_result.get("results", {})
            smart_keys_created = []

            for result_id, result_data in results.items():
                # Smart naming based on task content and result type
                smart_key = self._generate_smart_key(task_description, result_id, result_data)

                # Store full result
                self.variable_manager.set(smart_key, result_data)
                storage_summary.append(f"• {smart_key}")
                smart_keys_created.append(smart_key)

                # Store data separately for direct access
                if result_data.get("data"):
                    data_key = f"{smart_key}.data"
                    self.variable_manager.set(data_key, result_data["data"])
                    storage_summary.append(f"• {data_key} (direct access)")

                    # Store with generic access pattern
                    generic_data_key = f"results.{result_id}.data"
                    self.variable_manager.set(generic_data_key, result_data["data"])
                    storage_summary.append(f"• {generic_data_key} (standard access)")

            # Update comprehensive quick access index
            quick_access = {
                "latest_delegation": main_key,
                "latest_task": task_description,
                "timestamp": datetime.now().isoformat(),
                "loop": self.current_loop_count,
                "outline_step": getattr(self, 'current_outline_step', 0),
                "stored_keys": [item.replace("• ", "") for item in storage_summary],
                "smart_keys": smart_keys_created,
                "access_patterns": {
                    "main_result": main_key,
                    "by_loop": f"delegation.loop_{self.current_loop_count}",
                    "latest": "reasoning.latest_results",
                    "data_direct": [key for key in storage_summary if ".data" in key]
                }
            }
            self.variable_manager.set("reasoning.latest_results", quick_access)

            # Update global delegation index for easy discovery
            delegation_index = self.variable_manager.get("delegation.index", [])
            index_entry = {
                "loop": self.current_loop_count,
                "task": task_description[:100] + ("..." if len(task_description) > 100 else ""),
                "keys_created": len(storage_summary),
                "timestamp": datetime.now().isoformat(),
                "main_key": main_key,
                "smart_keys": smart_keys_created
            }
            delegation_index.append(index_entry)
            self.variable_manager.set("delegation.index", delegation_index[-50:])  # Keep last 50

            # Store task-specific quick access
            task_hash = hash(task_description) % 10000
            self.variable_manager.set(f"delegation.by_task.{task_hash}", {
                "task_description": task_description,
                "results": quick_access,
                "created_at": datetime.now().isoformat()
            })

            return f"\n📊 Auto-stored results ({len(storage_summary)} entries):\n" + "\n".join(storage_summary[:8]) + (
                f"\n... +{len(storage_summary) - 8} more" if len(storage_summary) > 8 else "")

        except Exception as e:
            return f"\n❌ Auto-storage failed: {str(e)}"

    def _generate_smart_key(self, task_description: str, result_id: str, result_data: dict) -> str:
        """Generate intelligent storage keys based on task content and result type"""
        task_lower = task_description.lower()

        # Analyze task type
        if "read" in task_lower and "file" in task_lower:
            prefix = "file_content"
        elif "write" in task_lower and "file" in task_lower:
            prefix = "file_written"
        elif "create" in task_lower and "file" in task_lower:
            prefix = "file_created"
        elif "search" in task_lower or "find" in task_lower:
            prefix = "search_results"
        elif "analyze" in task_lower or "analysis" in task_lower:
            prefix = "analysis_results"
        elif "list" in task_lower or "directory" in task_lower:
            prefix = "directory_listing"
        elif "download" in task_lower or "fetch" in task_lower:
            prefix = "downloaded_content"
        else:
            # Analyze result data for hints
            result_str = str(result_data).lower()
            if "file" in result_str and "content" in result_str:
                prefix = "file_content"
            elif "search" in result_str or "results" in result_str:
                prefix = "search_results"
            elif "data" in result_str:
                prefix = "task_data"
            else:
                prefix = "task_result"

        # Create unique key with loop and result ID
        return f"{prefix}.loop_{self.current_loop_count}_{result_id}"

    async def _mark_step_completion(self, prep_res: dict, method: str, evidence: str):
        """Mark current outline step as completed"""
        if not self.outline or not self.outline.get("steps"):
            return

        steps = self.outline["steps"]
        if self.current_outline_step < len(steps):
            current_step = steps[self.current_outline_step]
            current_step["status"] = "completed"
            current_step["completion_method"] = method
            current_step["completion_evidence"] = evidence
            current_step["completed_at"] = datetime.now().isoformat()

            # Store in variables
            if self.variable_manager:
                completion_data = {
                    "step_number": self.current_outline_step,
                    "description": current_step.get("description", ""),
                    "method": method,
                    "evidence": evidence,
                    "timestamp": datetime.now().isoformat()
                }
                self.variable_manager.set(f"reasoning.step_completions.{self.current_outline_step}", completion_data)

    async def _store_successful_completion(self, prep_res: dict, final_answer: str):
        """Store successful completion data for future learning"""
        if not self.variable_manager:
            return

        success_data = {
            "query": prep_res["original_query"],
            "final_answer": final_answer,
            "reasoning_loops": self.current_loop_count,
            "outline": self.outline,
            "performance_metrics": self.performance_metrics,
            "auto_recovery_attempts": self.auto_recovery_attempts,
            "completion_timestamp": datetime.now().isoformat(),
            "session_id": prep_res.get("session_id", "default")
        }

        # Store in successful patterns
        successes = self.variable_manager.get("reasoning.successful_patterns", [])
        successes.append(success_data)
        self.variable_manager.set("reasoning.successful_patterns", successes[-20:])  # Keep last 20

        # Update performance statistics
        self._update_success_statistics()

    def _update_success_statistics(self):
        """Update success statistics in variables"""
        if not self.variable_manager:
            return

        # Get current stats
        current_stats = self.variable_manager.get("reasoning.performance.statistics", {})

        # Update stats
        current_stats["total_successful_sessions"] = current_stats.get("total_successful_sessions", 0) + 1
        current_stats["avg_loops_per_success"] = current_stats.get("avg_loops_per_success", 0)

        # Calculate new average
        total_sessions = current_stats["total_successful_sessions"]
        old_avg = current_stats["avg_loops_per_success"] * (total_sessions - 1)
        current_stats["avg_loops_per_success"] = (old_avg + self.current_loop_count) / total_sessions

        # Store updated stats
        self.variable_manager.set("reasoning.performance.statistics", current_stats)

    async def _create_outline_completion_response(self, prep_res: dict) -> str:
        """Create response when outline is completed"""
        if not self.outline:
            return "Outline completion response requested but no outline available"

        steps = self.outline.get("steps", [])
        completed_steps = [s for s in steps if
                           s.get("status") in ["completed", "force_completed", "emergency_completed"]]

        response_parts = []
        response_parts.append("I have completed the structured approach outlined for your request:")

        # Summarize completed steps
        for i, step in enumerate(completed_steps):
            status_indicator = "✓" if step.get("status") == "completed" else "⚠️"
            response_parts.append(f"{status_indicator} Step {i + 1}: {step.get('description', 'Unknown step')}")

            # Add evidence if available
            evidence = step.get("completion_evidence", "")
            if evidence and len(evidence) < 200:
                response_parts.append(f"   Result: {evidence}")

        # Get final results from variables if available
        if self.variable_manager:
            final_results = self.variable_manager.get("reasoning.final_results", {})
            if final_results:
                response_parts.append("\nKey findings:")
                for key, value in final_results.items():
                    if isinstance(value, str) and len(value) < 300:
                        response_parts.append(f"- {key}: {value}")

        response_parts.append(
            f"\nCompleted in {self.current_loop_count} reasoning cycles using outline-driven execution.")

        return "\n".join(response_parts)

    async def _create_enhanced_timeout_response(self, query: str, prep_res: dict) -> str:
        """Create enhanced timeout response with comprehensive progress summary"""
        response_parts = []
        response_parts.append(
            f"I reached my reasoning limit of {self.max_reasoning_loops} steps while working on: {query}")

        # Outline progress
        if self.outline:
            steps = self.outline.get("steps", [])
            completed_steps = [s for s in steps if
                               s.get("status") in ["completed", "force_completed", "emergency_completed"]]
            unfinished_steps = [s for s in steps if s not in completed_steps]

            response_parts.append(f"\nOutline Progress: {len(completed_steps)}/{len(steps)} steps completed")

            if completed_steps:
                response_parts.append("Completed steps:")
                for i, step in enumerate(completed_steps):
                    response_parts.append(f"✓ {step.get('description', f'Step {i + 1}')}")

            if unfinished_steps:
                response_parts.append("Unfinished steps:")
                for i, step in enumerate(unfinished_steps):
                    response_parts.append(f"✗ {step.get('description', f'Step {i + 1}')}")

        # Task stack progress
        if self.internal_task_stack:
            completed_tasks = [t for t in self.internal_task_stack if t.get("status") == "completed"]
            pending_tasks = [t for t in self.internal_task_stack if t.get("status") == "pending"]

            response_parts.append(f"\nTask Progress: {len(completed_tasks)} completed, {len(pending_tasks)} pending")

        # Performance metrics
        if self.performance_metrics:
            response_parts.append(
                f"\nPerformance: {self.performance_metrics.get('action_efficiency', 0):.1%} efficiency, {self.auto_recovery_attempts} recovery attempts")

        # Available results from variables
        if self.variable_manager:
            reasoning_results = self.variable_manager.get("reasoning", {})
            if reasoning_results:
                response_parts.append(f"\nStored findings: {len(reasoning_results)} entries in reasoning variables")

        return "\n".join(response_parts)

    async def _finalize_reasoning_session(self, prep_res: dict, final_result: str):
        """Finalize reasoning session with comprehensive data storage"""
        if not self.variable_manager:
            return

        # Store session completion data
        session_data = {
            "query": prep_res["original_query"],
            "final_result": final_result,
            "reasoning_loops": self.current_loop_count,
            "outline_completion": self.current_outline_step,
            "performance_metrics": self.performance_metrics,
            "auto_recovery_attempts": self.auto_recovery_attempts,
            "context_summaries": len([c for c in self.reasoning_context if c.get("type") == "context_summary"]),
            "completion_timestamp": datetime.now().isoformat(),
            "session_duration": time.time() - time.mktime(datetime.now().timetuple()),
            "success": True
        }

        # Store in session history
        session_history = self.variable_manager.get("reasoning.session_history", [])
        session_history.append(session_data)
        self.variable_manager.set("reasoning.session_history", session_history[-50:])  # Keep last 50 sessions

        # Store outline pattern for reuse
        if self.outline:
            outline_pattern = {
                "query_type": self._classify_query_type(prep_res["original_query"]),
                "outline": self.outline,
                "success": True,
                "loops_used": self.current_loop_count,
                "timestamp": datetime.now().isoformat()
            }
            patterns = self.variable_manager.get("reasoning.successful_patterns.outlines", [])
            patterns.append(outline_pattern)
            self.variable_manager.set("reasoning.successful_patterns.outlines", patterns[-10:])

    def _classify_query_type(self, query: str) -> str:
        """Classify query type for pattern matching"""
        query_lower = query.lower()

        if any(word in query_lower for word in ["search", "find", "look up", "research"]):
            return "research"
        elif any(word in query_lower for word in ["analyze", "compare", "evaluate"]):
            return "analysis"
        elif any(word in query_lower for word in ["create", "generate", "write", "build"]):
            return "creation"
        elif any(word in query_lower for word in ["plan", "strategy", "approach"]):
            return "planning"
        else:
            return "general"

    async def _handle_reasoning_error(self, error: Exception, prep_res: dict, progress_tracker):
        """Enhanced error handling with auto-recovery"""
        eprint(f"Reasoning loop {self.current_loop_count} failed: {error}")

        # Store error in context
        self.reasoning_context.append({
            "type": "error",
            "content": f"Error in loop {self.current_loop_count}: {str(error)}",
            "error_type": type(error).__name__,
            "outline_step": self.current_outline_step,
            "timestamp": datetime.now().isoformat()
        })

        # Store in variables for learning
        if self.variable_manager:
            error_data = {
                "error": str(error),
                "error_type": type(error).__name__,
                "loop": self.current_loop_count,
                "outline_step": self.current_outline_step,
                "timestamp": datetime.now().isoformat(),
                "query": prep_res["original_query"]
            }
            errors = self.variable_manager.get("reasoning.error_log", [])
            errors.append(error_data)
            self.variable_manager.set("reasoning.error_log", errors[-100:])  # Keep last 100 errors

        # Trigger auto-recovery if not already in recovery
        if self.auto_recovery_attempts < self.max_auto_recovery:
            await self._trigger_auto_recovery(prep_res)

    # Keep all existing helper methods like _execute_internal_reasoning, etc.
    # but update them to use the enhanced variable system...

    async def post_async(self, shared, prep_res, exec_res):
        """Enhanced post-processing with comprehensive data storage"""
        final_result = exec_res.get("final_result", "Task processing incomplete")

        # Store comprehensive reasoning artifacts
        shared["reasoning_artifacts"] = {
            "reasoning_loops": exec_res.get("reasoning_loops", 0),
            "reasoning_context": exec_res.get("reasoning_context", []),
            "internal_task_stack": exec_res.get("internal_task_stack", []),
            "outline": exec_res.get("outline"),
            "outline_completion": exec_res.get("outline_completion", 0),
            "performance_metrics": exec_res.get("performance_metrics", {}),
            "auto_recovery_attempts": exec_res.get("auto_recovery_attempts", 0)
        }

        # Enhanced variable system updates
        if self.variable_manager:
            # Store final session results
            final_session_data = {
                "final_result": final_result,
                "completion_timestamp": datetime.now().isoformat(),
                "total_loops": exec_res.get("reasoning_loops", 0),
                "session_success": final_result != "Task processing incomplete",
                "outline_driven_execution": True
            }
            self.variable_manager.set("reasoning.current_session.final_data", final_session_data)

            # Update global performance statistics
            self._update_global_performance_stats(exec_res)

        # Set enhanced response data
        shared["llm_reasoner_result"] = final_result
        shared["current_response"] = final_result

        # Provide enhanced synthesis metadata
        shared["synthesized_response"] = {
            "synthesized_response": final_result,
            "confidence": self._calculate_confidence(exec_res),
            "metadata": {
                "synthesis_method": "outline_driven_reasoner",
                "reasoning_loops": exec_res.get("reasoning_loops", 0),
                "outline_completion": exec_res.get("outline_completion", 0),
                "performance_score": self._calculate_performance_score(exec_res),
                "auto_recovery_used": exec_res.get("auto_recovery_attempts", 0) > 0
            }
        }

        return "reasoner_complete"

    def _update_global_performance_stats(self, exec_res: dict):
        """Update global performance statistics in variables"""
        if not self.variable_manager:
            return

        stats = self.variable_manager.get("reasoning.global_performance", {})

        # Update counters
        stats["total_sessions"] = stats.get("total_sessions", 0) + 1
        stats["total_loops"] = stats.get("total_loops", 0) + exec_res.get("reasoning_loops", 0)
        stats["total_recoveries"] = stats.get("total_recoveries", 0) + exec_res.get("auto_recovery_attempts", 0)

        # Calculate averages
        stats["avg_loops_per_session"] = stats["total_loops"] / stats["total_sessions"]
        stats["recovery_rate"] = stats["total_recoveries"] / stats["total_sessions"]

        # Success tracking
        if exec_res.get("final_result") != "Task processing incomplete":
            stats["successful_sessions"] = stats.get("successful_sessions", 0) + 1
            stats["success_rate"] = stats["successful_sessions"] / stats["total_sessions"]

        self.variable_manager.set("reasoning.global_performance", stats)

    def _calculate_confidence(self, exec_res: dict) -> float:
        """Calculate confidence score based on execution results"""
        base_confidence = 0.5

        # Outline completion boosts confidence
        outline = exec_res.get("outline")
        if outline:
            completion_ratio = exec_res.get("outline_completion", 0) / len(outline.get("steps", [1]))
            base_confidence += 0.3 * completion_ratio

        # Low recovery attempts boost confidence
        recovery_attempts = exec_res.get("auto_recovery_attempts", 0)
        if recovery_attempts == 0:
            base_confidence += 0.15
        elif recovery_attempts == 1:
            base_confidence += 0.05

        # Reasonable loop count boosts confidence
        loops = exec_res.get("reasoning_loops", 0)
        if 3 <= loops <= 15:
            base_confidence += 0.1

        # Performance metrics
        performance = exec_res.get("performance_metrics", {})
        if performance.get("action_efficiency", 0) > 0.7:
            base_confidence += 0.1

        return min(1.0, max(0.0, base_confidence))

    def _calculate_performance_score(self, exec_res: dict) -> float:
        """Calculate overall performance score"""
        score = 0.5

        # Efficiency score
        performance = exec_res.get("performance_metrics", {})
        action_efficiency = performance.get("action_efficiency", 0)
        score += 0.3 * action_efficiency

        # Completion score
        outline = exec_res.get("outline")
        if outline:
            completion_ratio = exec_res.get("outline_completion", 0) / len(outline.get("steps", [1]))
            score += 0.4 * completion_ratio

        # Recovery penalty
        recovery_attempts = exec_res.get("auto_recovery_attempts", 0)
        score -= 0.1 * recovery_attempts

        return min(1.0, max(0.0, score))


    def _summarize_reasoning_context(self) -> str:
        """Summarize the current reasoning context"""
        if not self.reasoning_context:
            return "No previous reasoning steps"

        summary_parts = []
        for entry in self.reasoning_context[-5:]:  # Last 5 entries
            entry_type = entry.get("type", "unknown")
            content = entry.get("content", "")

            if entry_type == "reasoning":
                # Truncate long reasoning content
                content_preview = content[:20000] + "..." if len(content) > 20000 else content
                summary_parts.append(f"Loop {entry.get('loop', '?')}: {content_preview}")
            elif entry_type == "meta_tool_result":
                summary_parts.append(f"Result: {content[:150]}...")
            elif entry_type == "error":
                summary_parts.append(f"Error: {content}")

        return "\n".join(summary_parts)

    def _summarize_task_stack(self) -> str:
        """Summarize the internal task stack"""
        if not self.internal_task_stack:
            return "No tasks in stack"

        summary_parts = []
        for i, task in enumerate(self.internal_task_stack):
            status = task.get("status", "pending")
            description = task.get("description", "No description")
            summary_parts.append(f"{i + 1}. [{status.upper()}] {description}")

        return "\n".join(summary_parts)

    def _get_tool_category(self, tool_name: str) -> str:
        """Get category for meta-tool"""
        categories = {
            "internal_reasoning": "thinking",
            "manage_internal_task_stack": "planning",
            "delegate_to_llm_tool_node": "delegation",
            "create_and_execute_plan": "orchestration",
            "direct_response": "completion"
        }
        return categories.get(tool_name, "unknown")

    def _calculate_reasoning_depth(self) -> int:
        """Calculate current reasoning depth"""
        reasoning_entries = [entry for entry in self.reasoning_context if entry.get("type") == "reasoning"]
        return len(reasoning_entries)

    def _assess_delegation_complexity(self, args: dict) -> str:
        """Assess complexity of delegation task"""
        task_desc = args.get("task_description", "")
        tools_count = len(args.get("tools_list", []))

        if tools_count > 3 or len(task_desc) > 100:
            return "high"
        elif tools_count > 1 or len(task_desc) > 50:
            return "medium"
        else:
            return "low"

    def _estimate_plan_complexity(self, goals: list) -> str:
        """Estimate complexity of plan"""
        goal_count = len(goals)
        total_text = sum(len(str(goal)) for goal in goals)

        if goal_count > 5 or total_text > 500:
            return "high"
        elif goal_count > 2 or total_text > 200:
            return "medium"
        else:
            return "low"

    def _calculate_tool_performance_score(self, duration: float, tool_name: str) -> float:
        """Calculate performance score for tool execution"""
        # Expected durations by tool type
        expected_durations = {
            "internal_reasoning": 0.1,
            "manage_internal_task_stack": 0.05,
            "delegate_to_llm_tool_node": 3.0,
            "create_and_execute_plan": 10.0,
            "direct_response": 0.1
        }

        expected = expected_durations.get(tool_name, 1.0)
        if duration <= expected:
            return 1.0
        else:
            return max(0.0, expected / duration)

    def _create_reasoning_summary(self) -> str:
        """Create summary of reasoning process"""
        reasoning_entries = [entry for entry in self.reasoning_context if entry.get("type") == "reasoning"]
        task_entries = len(self.internal_task_stack)

        return f"Completed {len(reasoning_entries)} reasoning steps with {task_entries} tasks tracked"

    def _calculate_batch_performance(self, matches: list) -> dict[str, Any]:
        """Calculate performance metrics for batch execution"""
        tool_types = [match[0] for match in matches]
        return {
            "total_tools": len(matches),
            "tool_diversity": len(set(tool_types)),
            "most_used_tool": max(set(tool_types), key=tool_types.count) if tool_types else "none"
        }

    def _assess_reasoning_progress(self) -> str:
        """Assess overall reasoning progress"""
        if len(self.reasoning_context) < 3:
            return "early_stage"
        elif len(self.reasoning_context) < 8:
            return "developing"
        elif len(self.reasoning_context) < 15:
            return "mature"
        else:
            return "extensive"

    def _get_error_context(self, error: Exception) -> dict[str, Any]:
        """Get contextual information about an error"""
        return {
            "error_class": type(error).__name__,
            "reasoning_stage": f"loop_{self.current_loop_count}",
            "context_available": len(self.reasoning_context) > 0,
            "stack_state": "populated" if self.internal_task_stack else "empty"
        }

    async def _execute_internal_reasoning(self, args: dict, prep_res: dict) -> dict[str, Any]:
        """Execute internal reasoning meta-tool"""
        thought = args.get("thought", "")
        thought_number = args.get("thought_number", 1)
        total_thoughts = args.get("total_thoughts", 1)
        next_thought_needed = args.get("next_thought_needed", False)
        current_focus = args.get("current_focus", "")
        key_insights = args.get("key_insights", [])
        potential_issues = args.get("potential_issues", [])
        confidence_level = args.get("confidence_level", 0.5)

        # Structure the reasoning entry
        reasoning_entry = {
            "thought": thought,
            "thought_number": thought_number,
            "total_thoughts": total_thoughts,
            "next_thought_needed": next_thought_needed,
            "current_focus": current_focus,
            "key_insights": key_insights,
            "potential_issues": potential_issues,
            "confidence_level": confidence_level,
            "timestamp": datetime.now().isoformat()
        }

        # Add to internal reasoning log
        if not hasattr(self, 'internal_reasoning_log'):
            self.internal_reasoning_log = []
        self.internal_reasoning_log.append(reasoning_entry)

        # Format for context
        context_addition = f"""Internal Reasoning Step {thought_number}/{total_thoughts}:
Thought: {thought}
Focus: {current_focus}
Key Insights: {', '.join(key_insights) if key_insights else 'None'}
Potential Issues: {', '.join(potential_issues) if potential_issues else 'None'}
Confidence: {confidence_level}
Next Thought Needed: {next_thought_needed}"""

        return {"context_addition": context_addition}

    async def _execute_manage_task_stack(self, args: dict, prep_res: dict) -> dict[str, Any]:
        """Execute task stack management meta-tool"""
        action = args.get("action", "get_current").lower()
        task_description = args.get("task_description", "")

        if action == "add":
            self.internal_task_stack.append({
                "description": task_description,
                "status": "pending",
                "added_at": datetime.now().isoformat()
            })
            context_addition = f"Added to task stack: {task_description}"

        elif action == "remove":
            # Remove task by description match
            original_count = len(self.internal_task_stack)
            self.internal_task_stack = [
                task for task in self.internal_task_stack
                if task_description.lower() not in task["description"].lower()
            ]
            removed_count = original_count - len(self.internal_task_stack)
            context_addition = f"Removed {removed_count} task(s) matching: {task_description}"

        elif action == "complete":
            # Mark task as completed
            for task in self.internal_task_stack:
                if task_description.lower() in task["description"].lower():
                    task["status"] = "completed"
                    task["completed_at"] = datetime.now().isoformat()
            context_addition = f"Marked as completed: {task_description}"

        elif action == "get_current":
            if self.internal_task_stack:
                stack_summary = []
                for i, task in enumerate(self.internal_task_stack):
                    status = task["status"]
                    desc = task["description"]
                    stack_summary.append(f"{i + 1}. [{status.upper()}] {desc}")
                context_addition = "Current task stack:\n" + "\n".join(stack_summary)
            else:
                context_addition = "Task stack is empty"

        else:
            context_addition = f"Unknown task stack action: {action}"

        return {"context_addition": context_addition}

    async def _execute_delegate_llm_tool(self, args: dict, prep_res: dict) -> dict[str, Any]:
        """Execute delegation to LLMToolNode"""
        task_description = args.get("task_description", "")
        tools_list = args.get("tools_list", [])

        # Prepare shared state for LLMToolNode
        llm_tool_shared = {
            "current_task_description": task_description + '\nreturn all results in the final answer!',
            "current_query": task_description,
            "formatted_context": {
                "recent_interaction": f"Reasoner delegating task: {task_description}",
                "session_summary": self._get_reasoning_summary(),
                "task_context": f"Reasoning loop {self.current_loop_count}, delegated task. return all results!"
            },
            "variable_manager": prep_res.get("variable_manager"),
            "agent_instance": prep_res.get("agent_instance"),
            "available_tools": tools_list,  # Restrict to specific tools
            "tool_capabilities": prep_res.get("tool_capabilities", {}),
            "fast_llm_model": prep_res.get("fast_llm_model"),
            "complex_llm_model": prep_res.get("complex_llm_model"),
            "progress_tracker": prep_res.get("progress_tracker"),
            "session_id": prep_res.get("session_id"),
            "use_fast_response": True  # Use fast model for delegated tasks
        }

        # Execute LLMToolNode
        try:
            llm_tool_node = LLMToolNode()
            await llm_tool_node.run_async(llm_tool_shared)

            # Get results
            final_response = llm_tool_shared.get("current_response", "Task completed without specific result")
            tool_calls_made = llm_tool_shared.get("tool_calls_made", 0)

            context_addition = f"""Delegated Task Completed:
Task: {task_description}
Tools Available: {', '.join(tools_list)}
Tools Used: {tool_calls_made} tool calls made
Result: {final_response}"""

            return {"context_addition": context_addition}

        except Exception as e:
            context_addition = f"Delegation failed for task '{task_description}': {str(e)}"
            return {"context_addition": context_addition}

    async def _execute_create_plan(self, args: dict, prep_res: dict) -> dict[str, Any]:
        """Execute plan creation and execution"""
        goals = args.get("goals", [])

        if not goals:
            return {"context_addition": "No goals provided for plan creation"}

        try:
            # Prepare shared state for TaskPlanner
            planning_shared = prep_res.copy()
            planning_shared.update({
                "replan_context": {
                    "goals": goals,
                    "triggered_by": "llm_reasoner",
                    "reasoning_context": self._get_reasoning_summary()
                },
                "current_task_description": f"Execute plan with {len(goals)} goals",
                "current_query": f"Complex task: {'; '.join(goals)}"
            })

            # Execute TaskPlanner
            planner_node = TaskPlannerNode()
            plan_info = await planner_node.run_async(planning_shared)

            if plan_info == "planning_failed":
                return {"context_addition": f"Plan creation failed: {planning_shared.get('planning_error', 'Unknown error')}"}

            plan = planning_shared.get("current_plan")
            # Execute the plan using TaskExecutor
            executor_shared = planning_shared.copy()
            executor_node = TaskExecutorNode()

            # Execute plan to completion
            max_execution_cycles = 10
            execution_cycle = 0

            while execution_cycle < max_execution_cycles:
                execution_cycle += 1

                result = await executor_node.run_async(executor_shared)

                # Check completion status
                if result == "plan_completed" or result == "execution_error":
                    break
                elif result in ["continue_execution", "waiting"]:
                    continue
                else:
                    # Handle other results like reflection needs
                    if result in ["needs_dynamic_replan", "needs_plan_append"]:
                        # For now, just continue - could add reflection logic here
                        continue
                    break

            # Collect results
            completed_tasks = [
                task for task in plan.tasks
                if executor_shared["tasks"][task.id].status == "completed"
            ]

            failed_tasks = [
                task for task in plan.tasks
                if executor_shared["tasks"][task.id].status == "failed"
            ]

            # Build context addition with results
            results_summary = []
            results_store = executor_shared.get("results", {})

            for task in completed_tasks:
                task_result = results_store.get(task.id, {})
                if task_result.get("data"):
                    result_preview = str(task_result["data"])[:150] + "..."
                    results_summary.append(f"- {task.description}: {result_preview}")

            context_addition = f"""Plan Execution Completed:
Goals: {len(goals)} goals processed
Tasks Created: {len(plan.tasks)}
Tasks Completed: {len(completed_tasks)}
Tasks Failed: {len(failed_tasks)}
Execution Cycles: {execution_cycle}

Results Summary:
{chr(10).join(results_summary) if results_summary else 'No specific results captured'}"""

            return {"context_addition": context_addition}

        except Exception as e:
            import traceback
            print(traceback.format_exc())
            context_addition = f"Plan execution failed: {str(e)}"
            return {"context_addition": context_addition}

    def _get_reasoning_summary(self) -> str:
        """Get a summary of the reasoning process so far"""
        if not self.reasoning_context:
            return "No reasoning context available"

        summary_parts = []
        reasoning_entries = [entry for entry in self.reasoning_context if entry.get("type") == "reasoning"]

        for entry in reasoning_entries[-3:]:  # Last 3 reasoning steps
            content = entry.get("content", "")[:50000] + "..."
            loop_num = entry.get("loop", "?")
            summary_parts.append(f"Loop {loop_num}: {content}")

        return "\n".join(summary_parts)

    async def _create_error_response(self, query: str, error: str) -> str:
        """Create an error response"""
        return f"I encountered an error while processing your request: {error}. I was working on: {query}"

    async def _fallback_direct_response(self, prep_res: dict) -> dict[str, Any]:
        """Fallback when LLM is not available"""
        query = prep_res["original_query"]
        fallback_response = f"I received your request: {query}. However, I'm currently unable to process complex requests due to limited capabilities."

        return {
            "final_result": fallback_response,
            "reasoning_loops": 0,
            "reasoning_context": [{"type": "fallback", "content": "LLM unavailable"}],
            "internal_task_stack": []
        }

exec_async(prep_res) async

Enhanced main reasoning loop with outline-driven execution

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

async def exec_async(self, prep_res):
    """Enhanced main reasoning loop with outline-driven execution"""
    if not LITELLM_AVAILABLE:
        return await self._fallback_direct_response(prep_res)

    original_query = prep_res["original_query"]
    agent_instance = prep_res["agent_instance"]
    progress_tracker = prep_res.get("progress_tracker")

    # Initialize enhanced reasoning context
    await self._initialize_reasoning_session(prep_res, original_query)

    # STEP 1: MANDATORY OUTLINE CREATION
    if not self.outline:
        with Spinner("Creating initial outline..."):
            outline_result = await self._create_initial_outline(prep_res)
        if self.outline and len(self.outline.get("steps", [])) == 1:
            # fast llm respose on the input metoning tis is a direct respose and evalute if the input dosent need an outline
            print("Fast direct response triggered")
            response = await self.agent_instance.a_run_llm_completion(
                model=prep_res.get("fast_llm_model", "openrouter/anthropic/claude-3-haiku"),
                messages=[{"role": "user", "content": prep_res["original_query"]}],
                temperature=0.3,
                max_tokens=2048,
                node_name="LLMReasonerNode",
                task_id="fast_direct_response"
            )
            return {
                    "final_result": response,
                    "reasoning_loops": self.current_loop_count,
                    "reasoning_context": self.reasoning_context.copy(),
                    "internal_task_stack": self.internal_task_stack.copy(),
                    "outline": self.outline,
                    "outline_completion": self.current_outline_step,
                    "performance_metrics": self.performance_metrics,
                    "auto_recovery_attempts": self.auto_recovery_attempts
                }
        elif not outline_result:
            return await self._fallback_direct_response(prep_res)

    final_result = None
    consecutive_no_progress = 0
    max_no_progress = 3

    # Enhanced main reasoning loop with strict progress tracking
    while self.current_reasoning_count < self.max_reasoning_loops:
        self.current_loop_count += 1
        loop_start_time = time.time()

        # Check for infinite loops
        if self._detect_infinite_loop():
            await self._trigger_auto_recovery(prep_res)
            if self.auto_recovery_attempts >= self.max_auto_recovery:
                break

        # Auto-context management
        await self._manage_context_size()

        # Progress tracking
        if progress_tracker:
            await progress_tracker.emit_event(ProgressEvent(
                event_type="reasoning_loop",
                timestamp=time.time(),
                node_name="LLMReasonerNode",
                status=NodeStatus.RUNNING,
                metadata={
                    "loop_number": self.current_loop_count,
                    "outline_step": self.current_outline_step,
                    "outline_total": len(self.outline.get("steps", [])) if self.outline else 0,
                    "context_size": len(self.reasoning_context),
                    "task_stack_size": len(self.internal_task_stack),
                    "auto_recovery_attempts": self.auto_recovery_attempts,
                    "performance_metrics": self.performance_metrics
                }
            ))

        try:
            # Build enhanced reasoning prompt with outline context
            reasoning_prompt = await self._build_outline_driven_prompt(prep_res)

            # Force progress check if needed
            if self.mandatory_progress_check and consecutive_no_progress >= 2:
                reasoning_prompt += "\n\n**MANDATORY**: You must either complete current outline step or move to next step. No more analysis without action!"

            # LLM reasoning call
            model_to_use = prep_res.get("complex_llm_model", "openrouter/openai/gpt-4o")

            llm_response = await agent_instance.a_run_llm_completion(
                model=model_to_use,
                messages=[{"role": "user", "content": reasoning_prompt}],
                temperature=0.2,  # Lower temperature for more focused execution
                # max_tokens=3072,
                node_name="LLMReasonerNode",
                stop="<immediate_context>",
                task_id=f"reasoning_loop_{self.current_loop_count}_step_{self.current_outline_step}"
            )

            # Add LLM response to context
            self.reasoning_context.append({
                "type": "reasoning",
                "content": llm_response,
                "loop": self.current_loop_count,
                "outline_step": self.current_outline_step,
                "timestamp": datetime.now().isoformat()
            })

            # Parse and execute meta-tool calls with enhanced tracking
            progress_made = await self._parse_and_execute_meta_tools(llm_response, prep_res)

            action_taken = progress_made.get("action_taken", False)
            actual_progress = progress_made.get("progress_made", False)

            # Update performance with correct progress indication
            self._update_performance_metrics(loop_start_time, actual_progress)

            if not action_taken:
                self.current_reasoning_count += 1
                if self.current_outline_step > len(self.outline.get("steps", [])):
                    progress_made["final_result"] = llm_response
                    rprint("Final result reached forced by outline step count")
                if self.current_outline_step < len(self.outline.get("steps", [])) and self.outline.get("steps", [])[self.current_outline_step].get("is_final", False):
                    progress_made["final_result"] = llm_response
                    rprint("Final result reached forced by outline step count final step")
            else:
                self.current_reasoning_count -= 1

            # Check for final result
            if progress_made.get("final_result"):
                final_result = progress_made["final_result"]
                await self._finalize_reasoning_session(prep_res, final_result)
                break

            # Progress monitoring
            if progress_made.get("action_taken"):
                consecutive_no_progress = 0
                self._update_performance_metrics(loop_start_time, True)
            else:
                consecutive_no_progress += 1
                self._update_performance_metrics(loop_start_time, False)

            # Check outline completion
            if self.outline and self.current_outline_step >= len(self.outline.get("steps", []))+self.max_reasoning_loops:
                # All outline steps completed, force final response
                final_result = await self._create_outline_completion_response(prep_res)
                break

            # Emergency break for excessive no-progress
            if consecutive_no_progress >= max_no_progress:
                await self._trigger_auto_recovery(prep_res)

        except Exception as e:
            await self._handle_reasoning_error(e, prep_res, progress_tracker)
            import traceback
            print(traceback.format_exc())
            if self.auto_recovery_attempts >= self.max_auto_recovery:
                final_result = await self._create_error_response(original_query, str(e))
                break


    # If no final result after max loops, create a comprehensive summary
    if not final_result:
        final_result = await self._create_enhanced_timeout_response(original_query, prep_res)

    return {
        "final_result": final_result,
        "reasoning_loops": self.current_loop_count,
        "reasoning_context": self.reasoning_context.copy(),
        "internal_task_stack": self.internal_task_stack.copy(),
        "outline": self.outline,
        "outline_completion": self.current_outline_step,
        "performance_metrics": self.performance_metrics,
        "auto_recovery_attempts": self.auto_recovery_attempts
    }

post_async(shared, prep_res, exec_res) async

Enhanced post-processing with comprehensive data storage

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

async def post_async(self, shared, prep_res, exec_res):
    """Enhanced post-processing with comprehensive data storage"""
    final_result = exec_res.get("final_result", "Task processing incomplete")

    # Store comprehensive reasoning artifacts
    shared["reasoning_artifacts"] = {
        "reasoning_loops": exec_res.get("reasoning_loops", 0),
        "reasoning_context": exec_res.get("reasoning_context", []),
        "internal_task_stack": exec_res.get("internal_task_stack", []),
        "outline": exec_res.get("outline"),
        "outline_completion": exec_res.get("outline_completion", 0),
        "performance_metrics": exec_res.get("performance_metrics", {}),
        "auto_recovery_attempts": exec_res.get("auto_recovery_attempts", 0)
    }

    # Enhanced variable system updates
    if self.variable_manager:
        # Store final session results
        final_session_data = {
            "final_result": final_result,
            "completion_timestamp": datetime.now().isoformat(),
            "total_loops": exec_res.get("reasoning_loops", 0),
            "session_success": final_result != "Task processing incomplete",
            "outline_driven_execution": True
        }
        self.variable_manager.set("reasoning.current_session.final_data", final_session_data)

        # Update global performance statistics
        self._update_global_performance_stats(exec_res)

    # Set enhanced response data
    shared["llm_reasoner_result"] = final_result
    shared["current_response"] = final_result

    # Provide enhanced synthesis metadata
    shared["synthesized_response"] = {
        "synthesized_response": final_result,
        "confidence": self._calculate_confidence(exec_res),
        "metadata": {
            "synthesis_method": "outline_driven_reasoner",
            "reasoning_loops": exec_res.get("reasoning_loops", 0),
            "outline_completion": exec_res.get("outline_completion", 0),
            "performance_score": self._calculate_performance_score(exec_res),
            "auto_recovery_used": exec_res.get("auto_recovery_attempts", 0) > 0
        }
    }

    return "reasoner_complete"

prep_async(shared) async

Enhanced initialization with variable system integration

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

async def prep_async(self, shared):
    """Enhanced initialization with variable system integration"""
    # Reset for new execution
    self.reasoning_context = []
    self.internal_task_stack = []
    self.current_loop_count = 0
    self.current_reasoning_count = 0
    self.outline = None
    self.current_outline_step = 0
    self.step_completion_tracking = {}
    self.loop_detection_memory = []
    self.performance_metrics = {
        "loop_times": [],
        "progress_loops": 0,
        "total_loops": 0
    }
    self.auto_recovery_attempts = 0
    self.last_action_signatures = []

    self.agent_instance = shared.get("agent_instance")

    # Enhanced variable manager integration
    self.variable_manager = shared.get("variable_manager", self.agent_instance.variable_manager)
    context_manager = shared.get("context_manager")

    if self.variable_manager:
        # Store reasoning session context
        session_context = {
            "session_id": shared.get("session_id", "default"),
            "start_time": datetime.now().isoformat(),
            "query": shared.get("current_query", ""),
            "reasoning_mode": "outline_driven"
        }
        self.variable_manager.set("reasoning.current_session", session_context)
        # Load previous successful patterns from variables
        self._load_historical_patterns()

    #Build comprehensive system context via UnifiedContextManager
    system_context = await self._build_enhanced_system_context_unified(shared, context_manager)

    return {
        "original_query": shared.get("current_query", ""),
        "session_id": shared.get("session_id", "default"),
        "agent_instance": shared.get("agent_instance"),
        "variable_manager": self.variable_manager,
        "context_manager": context_manager,  #Context Manager Reference
        "system_context": system_context,
        "available_tools": shared.get("available_tools", []),
        "tool_capabilities": shared.get("tool_capabilities", {}),
        "fast_llm_model": shared.get("fast_llm_model"),
        "complex_llm_model": shared.get("complex_llm_model"),
        "progress_tracker": shared.get("progress_tracker"),
        "formatted_context": shared.get("formatted_context", {}),
        "historical_context": await self._get_historical_context_unified(context_manager, shared.get("session_id")),
        "capabilities_summary": shared.get("capabilities_summary", ""),
        # Sub-system references
        "llm_tool_node": shared.get("llm_tool_node_instance"),
        "task_planner": shared.get("task_planner_instance"),
        "task_executor": shared.get("task_executor_instance"),
    }

LLMTask dataclass

Bases: Task

Spezialisierter Task für LLM-Aufrufe

Source code in toolboxv2/mods/isaa/base/Agent/types.py

@dataclass
class LLMTask(Task):
    """Spezialisierter Task für LLM-Aufrufe"""
    llm_config: dict[str, Any] = field(default_factory=lambda: {
        "model_preference": "fast",  # "fast" | "complex"
        "temperature": 0.7,
        "max_tokens": 1024
    })
    prompt_template: str = ""
    context_keys: list[str] = field(default_factory=list)  # Keys aus shared state
    output_schema: dict  = None  # JSON Schema für Validierung

LLMToolNode

Bases: AsyncNode

Enhanced LLM tool with automatic tool calling and agent loop integration

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

@with_progress_tracking
class LLMToolNode(AsyncNode):
    """Enhanced LLM tool with automatic tool calling and agent loop integration"""

    def __init__(self, model: str = None, max_tool_calls: int = 5, **kwargs):
        super().__init__(**kwargs)
        self.model = model or os.getenv("COMPLEXMODEL", "openrouter/qwen/qwen3-code")
        self.max_tool_calls = max_tool_calls
        self.call_log = []

    async def prep_async(self, shared):
        context = shared.get("formatted_context", {})
        task_description = shared.get("current_task_description", shared.get("current_query", ""))

        # Variable Manager integration
        variable_manager = shared.get("variable_manager")
        agent_instance = shared.get("agent_instance")

        return {
            "task_description": task_description,
            "context": context,
            "context_manager": shared.get("context_manager"),
            "session_id": shared.get("session_id"),
            "variable_manager": variable_manager,
            "agent_instance": agent_instance,
            "available_tools": shared.get("available_tools", []),
            "tool_capabilities": shared.get("tool_capabilities", {}),
            "persona_config": shared.get("persona_config"),
            "base_system_message": variable_manager.format_text(agent_instance.amd.get_system_message_with_persona()),
            "recent_interaction": context.get("recent_interaction", ""),
            "session_summary": context.get("session_summary", ""),
            "task_context": context.get("task_context", ""),
            "fast_llm_model": shared.get("fast_llm_model"),
            "complex_llm_model": shared.get("complex_llm_model"),
            "progress_tracker": shared.get("progress_tracker"),
            "tool_call_count": 0
        }

    async def exec_async(self, prep_res):
        """Main execution with tool calling loop"""
        if not LITELLM_AVAILABLE:
            return await self._fallback_response(prep_res)

        progress_tracker = prep_res.get("progress_tracker")

        conversation_history = []
        tool_call_count = 0
        final_response = None
        model_to_use = "auto"
        total_llm_calls = 0
        total_cost = 0.0
        total_tokens = 0

        # Initial system message with tool awareness
        system_message = self._build_tool_aware_system_message(prep_res)

        # Initial user prompt with variable resolution
        initial_prompt = await self._build_context_aware_prompt(prep_res)
        conversation_history.append({"role": "user", "content":  prep_res["variable_manager"].format_text(initial_prompt)})
        runs = 0
        while tool_call_count < self.max_tool_calls:
            runs += 1
            # Get LLM response
            messages = [{"role": "system", "content": system_message + ( "\nfist look at the context and reason over you intal step." if runs == 1 else "")}] + conversation_history

            model_to_use = self._select_optimal_model(prep_res["task_description"], prep_res)

            llm_start = time.perf_counter()

            try:
                agent_instance = prep_res["agent_instance"]
                response = await agent_instance.a_run_llm_completion(
                    model=model_to_use,
                    messages=messages,
                    temperature=0.7,
                    stream=False,
                    # max_tokens=2048,
                    node_name="LLMToolNode", task_id="llm_phase_" + str(runs)
                )

                llm_response = response
                if not llm_response and  not final_response:
                    final_response = "I encountered an error while processing your request."
                    break


                # Check for tool calls
                tool_calls = self._extract_tool_calls(llm_response)

                llm_response = prep_res["variable_manager"].format_text(llm_response)
                conversation_history.append({"role": "assistant", "content": llm_response})


                if not tool_calls:
                    # No more tool calls, this is the final response
                    final_response = llm_response
                    break

                # Execute tool calls
                tool_results = await self._execute_tool_calls(tool_calls, prep_res)
                tool_call_count += len(tool_calls)

                # Add tool results to conversation
                tool_results_text = self._format_tool_results(tool_results)
                final_response = tool_results_text
                conversation_history.append({"role": "user",
                                             "content": f"Tool results:\n{tool_results_text}\n\nPlease continue with the next action do nor repeat or provide your final response."})

                # Update variable manager with tool results
                self._update_variables_with_results(tool_results, prep_res["variable_manager"])

            except Exception as e:
                llm_duration = time.perf_counter() - llm_start

                if progress_tracker:
                    await progress_tracker.emit_event(ProgressEvent(
                        event_type="llm_call",  # Konsistenter Event-Typ
                        node_name="LLMToolNode",
                        session_id=prep_res.get("session_id"),
                        status=NodeStatus.FAILED,
                        success=False,
                        duration=llm_duration,
                        llm_model=model_to_use,
                        error_details={
                            "message": str(e),
                            "type": type(e).__name__
                        },
                        metadata={"call_number": total_llm_calls + 1}
                    ))
                eprint(f"LLM tool execution failed: {e}")
                final_response = f"I encountered an error while processing: {str(e)}"
                import traceback
                print(traceback.format_exc())
                break


        return {
            "success": True,
            "final_response": final_response or "I was unable to complete the request.",
            "tool_calls_made": tool_call_count,
            "conversation_history": conversation_history,
            "model_used": model_to_use,
            "llm_statistics": {
                "total_calls": total_llm_calls,
                "total_cost": total_cost,
                "total_tokens": total_tokens
            }
        }

    def _build_tool_aware_system_message(self, prep_res: dict) -> str:
        """Build a unified intelligent, tool-aware system message with context and relevance analysis."""

        # Base system message
        base_message = prep_res.get("base_system_message", "You are a helpful AI assistant.")
        available_tools = prep_res.get("available_tools", [])
        tool_capabilities = prep_res.get("tool_capabilities", {})
        variable_manager = prep_res.get("variable_manager")
        context = prep_res.get("context", {})
        agent_instance = prep_res.get("agent_instance")
        query = prep_res.get('task_description', '').lower()

        base_message += ("\n\nAlways follow this action pattern"
                         "**THINK** -> **PLAN** -> **ACT** using tools!\n"
                         "all progress must be stored to ( variable system, memory, external services )!\n"
                         "if working on code or file based tasks, update and crate the files! sve result in file!\n")

        # --- Part 1: List available tools & capabilities ---
        if available_tools:
            base_message += f"\n\n## Available Tools\nYou have access to these tools: {', '.join(available_tools)}\n"
            base_message += "Results will be stored to results.{tool_name}.data"

            for tool_name in available_tools:
                if tool_name in tool_capabilities:
                    cap = tool_capabilities[tool_name]
                    base_message += f"\n**{tool_name}**: {cap.get('primary_function', 'No description')}"
                    use_cases = cap.get('use_cases', [])
                    if use_cases:
                        base_message += f"\n  Use cases: {', '.join(use_cases[:3])}"

            # base_message += "\n\n## Tool Usage\nTo use tools, respond with:\nTOOL_CALL: tool_name(arg1='value1', arg2='value2')\nYou can make multiple tool calls in one response."
            base_message += """
## Tool Usage
To use tools, respond with a YAML block:
```yaml
TOOL_CALLS:
  - tool: tool_name
    args:
      arg1: value1
      arg2: value2
  - tool: another_tool
    args:
      code: |
        def example():
            return "multi-line code"
      text: |
        Multi-line text
        with arbitrary content
```
You can call multiple tools in one response. Use | for multi-line strings containing code or complex text."""

        # --- Part 2: Add variable context ---
        if variable_manager:
            var_context = variable_manager.get_llm_variable_context()
            if var_context:
                base_message += f"\n\n## Variable Context\n{var_context}"

        # --- Part 3: Intelligent tool analysis ---
        if not agent_instance or not hasattr(agent_instance, '_tool_capabilities'):
            return base_message + "\n\n⚠ No intelligent tool analysis available."

        capabilities = agent_instance._tool_capabilities
        analysis_parts = ["\n\n## Intelligent Tool Analysis"]

        for tool_name, cap in capabilities.items():
            analysis_parts.append(f"\n{tool_name}{cap.get('args_schema', '()')}:")
            analysis_parts.append(f"- Function: {cap.get('primary_function', 'Unknown')}")

            # Calculate relevance score
            relevance_score = self._calculate_tool_relevance(query, cap)
            analysis_parts.append(f"- Query relevance: {relevance_score:.2f}")

            if relevance_score > 0.65:
                analysis_parts.append("- ⭐ HIGHLY RELEVANT - SHOULD USE THIS TOOL!")

            # Trigger phrase matching
            triggers = cap.get('trigger_phrases', [])
            matched_triggers = [t for t in triggers if t.lower() in query]
            if matched_triggers:
                analysis_parts.append(f"- Matched triggers: {matched_triggers}")

            # Show top use cases
            use_cases = cap.get('use_cases', [])[:3]
            if use_cases:
                analysis_parts.append(f"- Use cases: {', '.join(use_cases)}")

        # Combine everything into a final message
        return base_message + "\n"+ "\n".join(analysis_parts)

    def _calculate_tool_relevance(self, query: str, capabilities: dict) -> float:
        """Calculate how relevant a tool is to the current query"""

        query_words = set(query.lower().split())

        # Check trigger phrases
        trigger_score = 0.0
        triggers = capabilities.get('trigger_phrases', [])
        for trigger in triggers:
            trigger_words = set(trigger.lower().split())
            if trigger_words.intersection(query_words):
                trigger_score += 0.04
        # Check confidence triggers if available
        conf_triggers = capabilities.get('confidence_triggers', {})
        for phrase, confidence in conf_triggers.items():
            if phrase.lower() in query:
                trigger_score += confidence/10
        # Check indirect connections
        indirect = capabilities.get('indirect_connections', [])
        for connection in indirect:
            connection_words = set(connection.lower().split())
            if connection_words.intersection(query_words):
                trigger_score += 0.02
        return min(1.0, trigger_score)

    @staticmethod
    def _extract_tool_calls_custom(text: str) -> list[dict]:
        """Extract tool calls from LLM response"""

        tool_calls = []

        pattern = r'TOOL_CALL:'
        matches = _extract_meta_tool_calls(text, pattern)

        for tool_name, args_str in matches:
            try:
                # Parse arguments
                args = _parse_tool_args(args_str)
                tool_calls.append({
                    "tool_name": tool_name,
                    "arguments": args
                })
            except Exception as e:
                wprint(f"Failed to parse tool call {tool_name}: {e}")

        return tool_calls

    @staticmethod
    def _extract_tool_calls(text: str) -> list[dict]:
        """Extract tool calls from LLM response using YAML format"""
        import re

        import yaml

        tool_calls = []

        # Pattern to find YAML blocks with TOOL_CALLS
        yaml_pattern = r'```yaml\s*\n(.*?TOOL_CALLS:.*?)\n```'
        yaml_matches = re.findall(yaml_pattern, text, re.DOTALL | re.IGNORECASE)

        # Also try without code blocks for simpler cases
        if not yaml_matches:
            simple_pattern = r'TOOL_CALLS:\s*\n((?:.*\n)*?)(?=\n\S|\Z)'
            simple_matches = re.findall(simple_pattern, text, re.MULTILINE)
            if simple_matches:
                yaml_matches = [f"TOOL_CALLS:\n{match}" for match in simple_matches]

        for yaml_content in yaml_matches:
            try:
                # Parse YAML content
                parsed_yaml = yaml.safe_load(yaml_content)

                if not isinstance(parsed_yaml, dict) or 'TOOL_CALLS' not in parsed_yaml:
                    continue

                calls = parsed_yaml['TOOL_CALLS']
                if not isinstance(calls, list):
                    calls = [calls]  # Handle single tool call

                for call in calls:
                    if isinstance(call, dict) and 'tool' in call:
                        tool_call = {
                            "tool_name": call['tool'],
                            "arguments": call.get('args', {})
                        }
                        tool_calls.append(tool_call)

            except yaml.YAMLError as e:
                wprint(f"Failed to parse YAML tool calls: {e}")
            except Exception as e:
                wprint(f"Error processing tool calls: {e}")

        return tool_calls

    def _select_optimal_model(self, task_description: str, prep_res: dict) -> str:
        """Select optimal model based on task complexity and available resources"""
        complexity_score = self._estimate_task_complexity(task_description, prep_res)
        if complexity_score > 0.7:
            return prep_res.get("complex_llm_model", "openrouter/openai/gpt-4o")
        else:
            return prep_res.get("fast_llm_model", "openrouter/anthropic/claude-3-haiku")

    def _estimate_task_complexity(self, task_description: str, prep_res: dict) -> float:
        """Estimate task complexity based on description, length, and available tools"""
        # Simple heuristic: length + keyword matching + tool availability
        description_length_score = min(len(task_description) / 500, 1.0)  # cap at 1.0
        keywords = ["analyze", "research", "generate", "simulate", "complex", "deep", "strategy"]
        keyword_score = sum(1 for k in keywords if k in task_description.lower()) / len(keywords)
        tool_score = min(len(prep_res.get("available_tools", [])) / 10, 1.0)

        # Weighted sum
        complexity_score = (0.5 * description_length_score) + (0.3 * keyword_score) + (0.2 * tool_score)
        return round(complexity_score, 2)

    async def _fallback_response(self, prep_res: dict) -> dict:
        """Fallback response if LiteLLM is not available"""
        wprint("LiteLLM not available — using fallback response.")
        return {
            "success": False,
            "final_response": (
                "I'm unable to process this request fully right now because the LLM interface "
                "is not available. Please try again later or check system configuration."
            ),
            "tool_calls_made": 0,
            "conversation_history": [],
            "model_used": None
        }

    async def _execute_tool_calls(self, tool_calls: list[dict], prep_res: dict) -> list[dict]:
        """Execute tool calls via agent"""
        agent_instance = prep_res.get("agent_instance")
        variable_manager = prep_res.get("variable_manager")
        progress_tracker = prep_res.get("progress_tracker")

        results = []

        for tool_call in tool_calls:
            tool_name = tool_call["tool_name"]
            arguments = tool_call["arguments"]

            # Start tool tracking
            tool_start = time.perf_counter()

            if progress_tracker:
                await progress_tracker.emit_event(ProgressEvent(
                    event_type="tool_call",
                    timestamp=time.time(),
                    status=NodeStatus.RUNNING,
                    node_name="LLMToolNode",
                    tool_name=tool_name,
                    tool_args=arguments,
                    session_id=prep_res.get("session_id"),
                    metadata={"tool_call_initiated": True}
                ))

            try:
                # Resolve variables in arguments
                if variable_manager:
                    resolved_args = {}
                    for key, value in arguments.items():
                        if isinstance(value, str):
                            resolved_args[key] = variable_manager.format_text(value)
                        else:
                            resolved_args[key] = value
                else:
                    resolved_args = arguments

                # Execute via agent
                result = await agent_instance.arun_function(tool_name, **resolved_args)
                tool_duration = time.perf_counter() - tool_start
                variable_manager.set(f"results.{tool_name}.data", result)
                results.append({
                    "tool_name": tool_name,
                    "arguments": resolved_args,
                    "success": True,
                    "result": result
                })

            except Exception as e:
                tool_duration = time.perf_counter() - tool_start
                error_message = str(e)
                error_type = type(e).__name__
                import traceback
                print(traceback.format_exc())


                if progress_tracker:
                    await progress_tracker.emit_event(ProgressEvent(
                        event_type="tool_call",
                        timestamp=time.time(),
                        node_name="LLMToolNode",
                        status=NodeStatus.FAILED,
                        tool_name=tool_name,
                        tool_args=arguments,
                        duration=tool_duration,
                        success=False,
                        tool_error=error_message,
                        session_id=prep_res.get("session_id"),
                        metadata={
                            "error": error_message,
                            "error_message": error_message,
                            "error_type": error_type
                        }
                    ))

                    # FIXED: Also send generic error event for error log
                    await progress_tracker.emit_event(ProgressEvent(
                        event_type="error",
                        timestamp=time.time(),
                        node_name="LLMToolNode",
                        status=NodeStatus.FAILED,
                        success=False,
                        tool_name=tool_name,
                        metadata={
                            "error": error_message,
                            "error_message": error_message,
                            "error_type": error_type,
                            "source": "tool_execution",
                            "tool_name": tool_name,
                            "tool_args": arguments
                        }
                    ))
                eprint(f"Tool execution failed {tool_name}: {e}")
                results.append({
                    "tool_name": tool_name,
                    "arguments": arguments,
                    "success": False,
                    "error": str(e)
                })

        return results

    def _format_tool_results(self, results: list[dict]) -> str:
        """Format tool results for LLM"""
        formatted = []

        for result in results:
            if result["success"]:
                formatted.append(f"✓ {result['tool_name']}: {result['result']}")
            else:
                formatted.append(f"✗ {result['tool_name']}: ERROR - {result['error']}")

        return "\n".join(formatted)

    def _update_variables_with_results(self, results: list[dict], variable_manager):
        """Update variable manager with tool results"""
        if not variable_manager:
            return

        for i, result in enumerate(results):
            if result["success"]:
                tool_name = result['tool_name']
                result_data = result['result']

                # FIXED: Store result in proper variable paths
                variable_manager.set(f"results.{tool_name}.data", result_data)
                variable_manager.set(f"tools.{tool_name}.result", result_data)

                # Also store with index for multiple calls to same tool
                var_key = f"tool_result_{tool_name}_{i}"
                variable_manager.set(var_key, result_data)

    async def _build_context_aware_prompt(self, prep_res: dict) -> str:
        """Build context-aware prompt mit UnifiedContextManager Integration"""
        variable_manager = prep_res.get("variable_manager")
        agent_instance = prep_res.get("agent_instance")
        context = prep_res.get("context", {})

        #Get unified context if available
        context_manager = prep_res.get("context_manager")
        session_id = prep_res.get("session_id", "default")

        unified_context_parts = []

        if context_manager:
            try:
                # Get unified context für LLM Tool usage
                unified_context = await context_manager.build_unified_context(session_id, prep_res.get("task_description", ""))

                # Format unified context for LLM consumption
                chat_history = unified_context.get("chat_history", [])
                if chat_history:
                    unified_context_parts.append("## Recent Conversation from Session")
                    for msg in chat_history[-5:]:  # Last 5 messages
                        timestamp = msg.get('timestamp', '')[:19]
                        role = msg.get('role', 'unknown')
                        content = msg.get('content', '')[:300] + ("..." if len(msg.get('content', '')) > 300 else "")
                        unified_context_parts.append(f"[{timestamp}] {role}: {content}")

                # Execution state from unified context
                execution_state = unified_context.get("execution_state", {})
                if execution_state:
                    system_status = execution_state.get('system_status', 'unknown')
                    active_tasks = execution_state.get('active_tasks', [])
                    recent_completions = execution_state.get('recent_completions', [])

                    unified_context_parts.append("\n## Current System State")
                    unified_context_parts.append(f"Status: {system_status}")
                    if active_tasks:
                        unified_context_parts.append(f"Active Tasks: {len(active_tasks)}")
                    if recent_completions:
                        unified_context_parts.append(f"Recent Completions: {len(recent_completions)}")

                # Available results from unified context
                variables_context = unified_context.get("variables", {})
                recent_results = variables_context.get("recent_results", [])
                if recent_results:
                    unified_context_parts.append("\n## Available Results")
                    for result in recent_results[:3]:  # Top 3 results
                        task_id = result.get("task_id", "unknown")
                        preview = result.get("preview", "")[:100] + "..."
                        success = "✅" if result.get("success") else "❌"
                        unified_context_parts.append(f"{success=} {task_id}: {preview=}")

                # World model facts from unified context
                relevant_facts = unified_context.get("relevant_facts", [])
                if relevant_facts:
                    unified_context_parts.append("\n## Relevant Known Facts")
                    for key, value in relevant_facts:  # Top 3 facts
                        fact_preview = str(value)
                        unified_context_parts.append(f"- {key}: {fact_preview}")

            except Exception as e:
                unified_context_parts.append(f"## Context Error\nUnified context unavailable: {str(e)}")

        # EXISTIEREND: Keep existing context building (backwards compatibility)
        prompt_parts = []

        # Add unified context first (primary)
        if unified_context_parts:
            prompt_parts.extend(unified_context_parts)

        # Add existing context sections (secondary)
        recent_interaction = prep_res.get("recent_interaction", "")
        session_summary = prep_res.get("session_summary", "")
        task_context = prep_res.get("task_context", "")

        if recent_interaction:
            prompt_parts.append(f"\n## Recent Interaction Context\n{recent_interaction}")
            prompt_parts.append("\n**Important**: NO META_TOOL_CALLs needed in this section! and not avalabel\n use tools from Intelligent Tool Analysis only!")
        if session_summary:
            prompt_parts.append(f"\n## Session Summary\n{session_summary}")
        if task_context:
            prompt_parts.append(f"\n## Task Context\n{task_context}")

        # Add main task
        task_description = prep_res.get("task_description", "")
        if task_description:
            prompt_parts.append(f"\n## Current Request\n{task_description}")

        # Variable suggestions (existing functionality)
        if variable_manager and task_description:
            suggestions = variable_manager.get_variable_suggestions(task_description)
            if suggestions:
                prompt_parts.append(f"\n## Available Variables\nYou can use: {', '.join(suggestions)}")

        # Final variable resolution
        final_prompt = "\n".join(prompt_parts)
        if variable_manager:
            final_prompt = variable_manager.format_text(final_prompt)

        return final_prompt

    async def post_async(self, shared, prep_res, exec_res):
        shared["current_response"] = exec_res.get("final_response", "Task completed.")
        shared["tool_calls_made"] = exec_res.get("tool_calls_made", 0)
        shared["llm_tool_conversation"] = exec_res.get("conversation_history", [])
        shared["synthesized_response"] = {"synthesized_response":exec_res.get("final_response", "Task completed."),
                                          "confidence": (0.7 if exec_res.get("model_used") == prep_res.get("complex_llm_model") else 0.6) if exec_res.get("success", False) else 0,
                                          "metadata": exec_res.get("metadata", {"model_used": exec_res.get("model_used")}),
                                          "synthesis_method": "llm_tool"}
        return "llm_tool_complete"

exec_async(prep_res) async

Main execution with tool calling loop

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

async def exec_async(self, prep_res):
    """Main execution with tool calling loop"""
    if not LITELLM_AVAILABLE:
        return await self._fallback_response(prep_res)

    progress_tracker = prep_res.get("progress_tracker")

    conversation_history = []
    tool_call_count = 0
    final_response = None
    model_to_use = "auto"
    total_llm_calls = 0
    total_cost = 0.0
    total_tokens = 0

    # Initial system message with tool awareness
    system_message = self._build_tool_aware_system_message(prep_res)

    # Initial user prompt with variable resolution
    initial_prompt = await self._build_context_aware_prompt(prep_res)
    conversation_history.append({"role": "user", "content":  prep_res["variable_manager"].format_text(initial_prompt)})
    runs = 0
    while tool_call_count < self.max_tool_calls:
        runs += 1
        # Get LLM response
        messages = [{"role": "system", "content": system_message + ( "\nfist look at the context and reason over you intal step." if runs == 1 else "")}] + conversation_history

        model_to_use = self._select_optimal_model(prep_res["task_description"], prep_res)

        llm_start = time.perf_counter()

        try:
            agent_instance = prep_res["agent_instance"]
            response = await agent_instance.a_run_llm_completion(
                model=model_to_use,
                messages=messages,
                temperature=0.7,
                stream=False,
                # max_tokens=2048,
                node_name="LLMToolNode", task_id="llm_phase_" + str(runs)
            )

            llm_response = response
            if not llm_response and  not final_response:
                final_response = "I encountered an error while processing your request."
                break


            # Check for tool calls
            tool_calls = self._extract_tool_calls(llm_response)

            llm_response = prep_res["variable_manager"].format_text(llm_response)
            conversation_history.append({"role": "assistant", "content": llm_response})


            if not tool_calls:
                # No more tool calls, this is the final response
                final_response = llm_response
                break

            # Execute tool calls
            tool_results = await self._execute_tool_calls(tool_calls, prep_res)
            tool_call_count += len(tool_calls)

            # Add tool results to conversation
            tool_results_text = self._format_tool_results(tool_results)
            final_response = tool_results_text
            conversation_history.append({"role": "user",
                                         "content": f"Tool results:\n{tool_results_text}\n\nPlease continue with the next action do nor repeat or provide your final response."})

            # Update variable manager with tool results
            self._update_variables_with_results(tool_results, prep_res["variable_manager"])

        except Exception as e:
            llm_duration = time.perf_counter() - llm_start

            if progress_tracker:
                await progress_tracker.emit_event(ProgressEvent(
                    event_type="llm_call",  # Konsistenter Event-Typ
                    node_name="LLMToolNode",
                    session_id=prep_res.get("session_id"),
                    status=NodeStatus.FAILED,
                    success=False,
                    duration=llm_duration,
                    llm_model=model_to_use,
                    error_details={
                        "message": str(e),
                        "type": type(e).__name__
                    },
                    metadata={"call_number": total_llm_calls + 1}
                ))
            eprint(f"LLM tool execution failed: {e}")
            final_response = f"I encountered an error while processing: {str(e)}"
            import traceback
            print(traceback.format_exc())
            break


    return {
        "success": True,
        "final_response": final_response or "I was unable to complete the request.",
        "tool_calls_made": tool_call_count,
        "conversation_history": conversation_history,
        "model_used": model_to_use,
        "llm_statistics": {
            "total_calls": total_llm_calls,
            "total_cost": total_cost,
            "total_tokens": total_tokens
        }
    }

PersonaConfig dataclass

Source code in toolboxv2/mods/isaa/base/Agent/types.py

@dataclass
class PersonaConfig:
    name: str
    style: str = "professional"
    personality_traits: list[str] = field(default_factory=lambda: ["helpful", "concise"])
    tone: str = "friendly"
    response_format: str = "direct"
    custom_instructions: str = ""

    format_config: FormatConfig  = None

    apply_method: str = "system_prompt"  # "system_prompt" | "post_process" | "both"
    integration_level: str = "light"  # "light" | "medium" | "heavy"

    def to_system_prompt_addition(self) -> str:
        """Convert persona to system prompt addition with format integration"""
        if self.apply_method in ["system_prompt", "both"]:
            additions = []
            additions.append(f"You are {self.name}.")
            additions.append(f"Your communication style is {self.style} with a {self.tone} tone.")

            if self.personality_traits:
                traits_str = ", ".join(self.personality_traits)
                additions.append(f"Your key traits are: {traits_str}.")

            if self.custom_instructions:
                additions.append(self.custom_instructions)

            # Format-spezifische Anweisungen hinzufügen
            if self.format_config:
                additions.append("\n" + self.format_config.get_combined_instructions())

            return " ".join(additions)
        return ""

    def update_format(self, response_format: ResponseFormat|str, text_length: TextLength|str, custom_instructions: str = ""):
        """Dynamische Format-Aktualisierung"""
        try:
            format_enum = ResponseFormat(response_format) if isinstance(response_format, str) else response_format
            length_enum = TextLength(text_length) if isinstance(text_length, str) else text_length

            if not self.format_config:
                self.format_config = FormatConfig()

            self.format_config.response_format = format_enum
            self.format_config.text_length = length_enum

            if custom_instructions:
                self.format_config.custom_instructions = custom_instructions


        except ValueError:
            raise ValueError(f"Invalid format '{response_format}' or length '{text_length}'")

    def should_post_process(self) -> bool:
        """Check if post-processing should be applied"""
        return self.apply_method in ["post_process", "both"]

should_post_process()

Check if post-processing should be applied

Source code in toolboxv2/mods/isaa/base/Agent/types.py

def should_post_process(self) -> bool:
    """Check if post-processing should be applied"""
    return self.apply_method in ["post_process", "both"]

to_system_prompt_addition()

Convert persona to system prompt addition with format integration

Source code in toolboxv2/mods/isaa/base/Agent/types.py

def to_system_prompt_addition(self) -> str:
    """Convert persona to system prompt addition with format integration"""
    if self.apply_method in ["system_prompt", "both"]:
        additions = []
        additions.append(f"You are {self.name}.")
        additions.append(f"Your communication style is {self.style} with a {self.tone} tone.")

        if self.personality_traits:
            traits_str = ", ".join(self.personality_traits)
            additions.append(f"Your key traits are: {traits_str}.")

        if self.custom_instructions:
            additions.append(self.custom_instructions)

        # Format-spezifische Anweisungen hinzufügen
        if self.format_config:
            additions.append("\n" + self.format_config.get_combined_instructions())

        return " ".join(additions)
    return ""

update_format(response_format, text_length, custom_instructions='')

Dynamische Format-Aktualisierung

Source code in toolboxv2/mods/isaa/base/Agent/types.py

def update_format(self, response_format: ResponseFormat|str, text_length: TextLength|str, custom_instructions: str = ""):
    """Dynamische Format-Aktualisierung"""
    try:
        format_enum = ResponseFormat(response_format) if isinstance(response_format, str) else response_format
        length_enum = TextLength(text_length) if isinstance(text_length, str) else text_length

        if not self.format_config:
            self.format_config = FormatConfig()

        self.format_config.response_format = format_enum
        self.format_config.text_length = length_enum

        if custom_instructions:
            self.format_config.custom_instructions = custom_instructions


    except ValueError:
        raise ValueError(f"Invalid format '{response_format}' or length '{text_length}'")

PlanData

Bases: BaseModel

Dataclass for plan data

Source code in toolboxv2/mods/isaa/base/Agent/types.py

class PlanData(BaseModel):
    """Dataclass for plan data"""
    plan_name: str = Field(..., discription="Name of the plan")
    description: str = Field(..., discription="Description of the plan")
    execution_strategy: str = Field(..., discription="Execution strategy for the plan")
    tasks: list[LLMTask | ToolTask | DecisionTask] = Field(..., discription="List of tasks in the plan")

ProgressEvent dataclass

Enhanced progress event with better error handling

Source code in toolboxv2/mods/isaa/base/Agent/types.py

@dataclass
class ProgressEvent:

    """Enhanced progress event with better error handling"""

    # === 1. Kern-Attribute (Für jedes Event) ===
    event_type: str
    node_name: str
    timestamp: float = field(default_factory=time.time)
    event_id: str = field(default_factory=lambda: str(uuid.uuid4()))
    session_id: Optional[str] = None

    # === 2. Status und Ergebnis-Attribute ===
    status: Optional[NodeStatus] = None
    success: Optional[bool] = None
    duration: Optional[float] = None
    error_details: dict[str, Any] = field(default_factory=dict)  # Strukturiert: message, type, traceback

    # === 3. LLM-spezifische Attribute ===
    llm_model: Optional[str] = None
    llm_prompt_tokens: Optional[int] = None
    llm_completion_tokens: Optional[int] = None
    llm_total_tokens: Optional[int] = None
    llm_cost: Optional[float] = None
    llm_input: Optional[Any] = None  # Optional für Debugging, kann groß sein
    llm_output: Optional[str] = None # Optional für Debugging, kann groß sein

    # === 4. Tool-spezifische Attribute ===
    tool_name: Optional[str] = None
    is_meta_tool: Optional[bool] = None
    tool_args: Optional[dict[str, Any]] = None
    tool_result: Optional[Any] = None
    tool_error: Optional[str] = None
    llm_temperature: Optional[float]  = None

    # === 5. Strategie- und Kontext-Attribute ===
    agent_name: Optional[str] = None
    task_id: Optional[str] = None
    plan_id: Optional[str] = None


    # Node/Routing data
    routing_decision: Optional[str] = None
    node_phase: Optional[str] = None
    node_duration: Optional[float] = None

    # === 6. Metadaten (Für alles andere) ===
    metadata: dict[str, Any] = field(default_factory=dict)


    def __post_init__(self):

        if self.timestamp is None:
            self.timestamp = time.time()

        if self.metadata is None:
            self.metadata = {}
        if not self.event_id:
            self.event_id = f"{self.node_name}_{self.event_type}_{int(self.timestamp * 1000000)}"
        if 'error' in self.metadata or 'error_type' in self.metadata:
            if self.error_details is None:
                self.error_details = {}
            self.error_details['error'] = self.metadata.get('error')
            self.error_details['error_type'] = self.metadata.get('error_type')
            self.status = NodeStatus.FAILED
        if self.status == NodeStatus.FAILED:
            self.success = False
        if self.status == NodeStatus.COMPLETED:
            self.success = True

    def _to_dict(self) -> dict[str, Any]:
        """Convert ProgressEvent to dictionary with proper handling of all field types"""
        result = {}

        # Get all fields from the dataclass
        for field in fields(self):
            value = getattr(self, field.name)

            # Handle None values
            if value is None:
                result[field.name] = None
                continue

            # Handle NodeStatus enum
            if isinstance(value, NodeStatus | Enum):
                result[field.name] = value.value
            # Handle dataclass objects
            elif is_dataclass(value):
                result[field.name] = asdict(value)
            # Handle dictionaries (recursively process nested enums/dataclasses)
            elif isinstance(value, dict):
                result[field.name] = self._process_dict(value)
            # Handle lists (recursively process nested items)
            elif isinstance(value, list):
                result[field.name] = self._process_list(value)
            # Handle primitive types
            else:
                result[field.name] = value

        return result

    def _process_dict(self, d: dict[str, Any]) -> dict[str, Any]:
        """Recursively process dictionary values"""
        result = {}
        for k, v in d.items():
            if isinstance(v, Enum):
                result[k] = v.value
            elif is_dataclass(v):
                result[k] = asdict(v)
            elif isinstance(v, dict):
                result[k] = self._process_dict(v)
            elif isinstance(v, list):
                result[k] = self._process_list(v)
            else:
                result[k] = v
        return result

    def _process_list(self, lst: list[Any]) -> list[Any]:
        """Recursively process list items"""
        result = []
        for item in lst:
            if isinstance(item, Enum):
                result.append(item.value)
            elif is_dataclass(item):
                result.append(asdict(item))
            elif isinstance(item, dict):
                result.append(self._process_dict(item))
            elif isinstance(item, list):
                result.append(self._process_list(item))
            else:
                result.append(item)
        return result

    @classmethod
    def from_dict(cls, data: dict[str, Any]) -> 'ProgressEvent':
        """Create ProgressEvent from dictionary"""
        # Create a copy to avoid modifying the original
        data_copy = dict(data)

        # Handle NodeStatus enum conversion from string back to enum
        if 'status' in data_copy and data_copy['status'] is not None:
            if isinstance(data_copy['status'], str):
                try:
                    data_copy['status'] = NodeStatus(data_copy['status'])
                except (ValueError, TypeError):
                    # If invalid status value, set to None
                    data_copy['status'] = None

        # Filter out any keys that aren't valid dataclass fields
        field_names = {field.name for field in fields(cls)}
        filtered_data = {k: v for k, v in data_copy.items() if k in field_names}

        # Ensure metadata is properly initialized
        if 'metadata' not in filtered_data or filtered_data['metadata'] is None:
            filtered_data['metadata'] = {}

        return cls(**filtered_data)

    def to_dict(self) -> dict[str, Any]:
        """Return event data with None values removed for compact display"""
        data = self._to_dict()

        def clean_dict(d):
            if isinstance(d, dict):
                return {k: clean_dict(v) for k, v in d.items()
                        if v is not None and v != {} and v != [] and v != ''}
            elif isinstance(d, list):
                cleaned_list = [clean_dict(item) for item in d if item is not None]
                return [item for item in cleaned_list if item != {} and item != []]
            return d

        return clean_dict(data)

    def get_chat_display_data(self) -> dict[str, Any]:
        """Get data optimized for chat view display"""
        filtered = self.filter_none_values()

        # Core fields always shown
        core_data = {
            'event_type': filtered.get('event_type'),
            'node_name': filtered.get('node_name'),
            'timestamp': filtered.get('timestamp'),
            'event_id': filtered.get('event_id'),
            'status': filtered.get('status')
        }

        # Add specific fields based on event type
        if self.event_type == 'outline_created':
            if 'metadata' in filtered:
                core_data['outline_steps'] = len(filtered['metadata'].get('outline', []))
        elif self.event_type == 'reasoning_loop':
            if 'metadata' in filtered:
                core_data.update({
                    'loop_number': filtered['metadata'].get('loop_number'),
                    'outline_step': filtered['metadata'].get('outline_step'),
                    'context_size': filtered['metadata'].get('context_size')
                })
        elif self.event_type == 'tool_call':
            core_data.update({
                'tool_name': filtered.get('tool_name'),
                'is_meta_tool': filtered.get('is_meta_tool')
            })
        elif self.event_type == 'llm_call':
            core_data.update({
                'llm_model': filtered.get('llm_model'),
                'llm_total_tokens': filtered.get('llm_total_tokens'),
                'llm_cost': filtered.get('llm_cost')
            })

        # Remove None values from core_data
        return {k: v for k, v in core_data.items() if v is not None}

    def get_detailed_display_data(self) -> dict[str, Any]:
        """Get complete filtered data for detailed popup view"""
        return self.filter_none_values()

    def get_progress_summary(self) -> str:
        """Get a brief summary for progress sidebar"""
        if self.event_type == 'reasoning_loop' and 'metadata' in self.filter_none_values():
            metadata = self.filter_none_values()['metadata']
            loop_num = metadata.get('loop_number', '?')
            step = metadata.get('outline_step', '?')
            return f"Loop {loop_num}, Step {step}"
        elif self.event_type == 'tool_call':
            tool_name = self.tool_name or 'Unknown Tool'
            return f"{'Meta ' if self.is_meta_tool else ''}{tool_name}"
        elif self.event_type == 'llm_call':
            model = self.llm_model or 'Unknown Model'
            tokens = self.llm_total_tokens
            return f"{model} ({tokens} tokens)" if tokens else model
        else:
            return self.event_type.replace('_', ' ').title()

from_dict(data) classmethod

Create ProgressEvent from dictionary

Source code in toolboxv2/mods/isaa/base/Agent/types.py

@classmethod
def from_dict(cls, data: dict[str, Any]) -> 'ProgressEvent':
    """Create ProgressEvent from dictionary"""
    # Create a copy to avoid modifying the original
    data_copy = dict(data)

    # Handle NodeStatus enum conversion from string back to enum
    if 'status' in data_copy and data_copy['status'] is not None:
        if isinstance(data_copy['status'], str):
            try:
                data_copy['status'] = NodeStatus(data_copy['status'])
            except (ValueError, TypeError):
                # If invalid status value, set to None
                data_copy['status'] = None

    # Filter out any keys that aren't valid dataclass fields
    field_names = {field.name for field in fields(cls)}
    filtered_data = {k: v for k, v in data_copy.items() if k in field_names}

    # Ensure metadata is properly initialized
    if 'metadata' not in filtered_data or filtered_data['metadata'] is None:
        filtered_data['metadata'] = {}

    return cls(**filtered_data)

get_chat_display_data()

Get data optimized for chat view display

Source code in toolboxv2/mods/isaa/base/Agent/types.py

def get_chat_display_data(self) -> dict[str, Any]:
    """Get data optimized for chat view display"""
    filtered = self.filter_none_values()

    # Core fields always shown
    core_data = {
        'event_type': filtered.get('event_type'),
        'node_name': filtered.get('node_name'),
        'timestamp': filtered.get('timestamp'),
        'event_id': filtered.get('event_id'),
        'status': filtered.get('status')
    }

    # Add specific fields based on event type
    if self.event_type == 'outline_created':
        if 'metadata' in filtered:
            core_data['outline_steps'] = len(filtered['metadata'].get('outline', []))
    elif self.event_type == 'reasoning_loop':
        if 'metadata' in filtered:
            core_data.update({
                'loop_number': filtered['metadata'].get('loop_number'),
                'outline_step': filtered['metadata'].get('outline_step'),
                'context_size': filtered['metadata'].get('context_size')
            })
    elif self.event_type == 'tool_call':
        core_data.update({
            'tool_name': filtered.get('tool_name'),
            'is_meta_tool': filtered.get('is_meta_tool')
        })
    elif self.event_type == 'llm_call':
        core_data.update({
            'llm_model': filtered.get('llm_model'),
            'llm_total_tokens': filtered.get('llm_total_tokens'),
            'llm_cost': filtered.get('llm_cost')
        })

    # Remove None values from core_data
    return {k: v for k, v in core_data.items() if v is not None}

get_detailed_display_data()

Get complete filtered data for detailed popup view

Source code in toolboxv2/mods/isaa/base/Agent/types.py

def get_detailed_display_data(self) -> dict[str, Any]:
    """Get complete filtered data for detailed popup view"""
    return self.filter_none_values()

get_progress_summary()

Get a brief summary for progress sidebar

Source code in toolboxv2/mods/isaa/base/Agent/types.py

def get_progress_summary(self) -> str:
    """Get a brief summary for progress sidebar"""
    if self.event_type == 'reasoning_loop' and 'metadata' in self.filter_none_values():
        metadata = self.filter_none_values()['metadata']
        loop_num = metadata.get('loop_number', '?')
        step = metadata.get('outline_step', '?')
        return f"Loop {loop_num}, Step {step}"
    elif self.event_type == 'tool_call':
        tool_name = self.tool_name or 'Unknown Tool'
        return f"{'Meta ' if self.is_meta_tool else ''}{tool_name}"
    elif self.event_type == 'llm_call':
        model = self.llm_model or 'Unknown Model'
        tokens = self.llm_total_tokens
        return f"{model} ({tokens} tokens)" if tokens else model
    else:
        return self.event_type.replace('_', ' ').title()

to_dict()

Return event data with None values removed for compact display

Source code in toolboxv2/mods/isaa/base/Agent/types.py

def to_dict(self) -> dict[str, Any]:
    """Return event data with None values removed for compact display"""
    data = self._to_dict()

    def clean_dict(d):
        if isinstance(d, dict):
            return {k: clean_dict(v) for k, v in d.items()
                    if v is not None and v != {} and v != [] and v != ''}
        elif isinstance(d, list):
            cleaned_list = [clean_dict(item) for item in d if item is not None]
            return [item for item in cleaned_list if item != {} and item != []]
        return d

    return clean_dict(data)

ProgressTracker

Advanced progress tracking with cost calculation

Source code in toolboxv2/mods/isaa/base/Agent/types.py

class ProgressTracker:
    """Advanced progress tracking with cost calculation"""

    def __init__(self, progress_callback: callable  = None, agent_name="unknown"):
        self.progress_callback = progress_callback
        self.events: list[ProgressEvent] = []
        self.active_timers: dict[str, float] = {}

        # Cost tracking (simplified - would need actual provider pricing)
        self.token_costs = {
            "input": 0.00001,  # $0.01/1K tokens input
            "output": 0.00003,  # $0.03/1K tokens output
        }
        self.agent_name = agent_name

    async def emit_event(self, event: ProgressEvent):
        """Emit progress event with callback and storage"""
        self.events.append(event)
        event.agent_name = self.agent_name

        if self.progress_callback:
            try:
                if asyncio.iscoroutinefunction(self.progress_callback):
                    await self.progress_callback(event)
                else:
                    self.progress_callback(event)
            except Exception:
                import traceback
                print(traceback.format_exc())


    def start_timer(self, key: str) -> float:
        """Start timing operation"""
        start_time = time.perf_counter()
        self.active_timers[key] = start_time
        return start_time

    def end_timer(self, key: str) -> float:
        """End timing operation and return duration"""
        if key not in self.active_timers:
            return 0.0
        duration = time.perf_counter() - self.active_timers[key]
        del self.active_timers[key]
        return duration

    def calculate_llm_cost(self, model: str, input_tokens: int, output_tokens: int,completion_response:Any=None) -> float:
        """Calculate approximate LLM cost"""
        cost = (input_tokens / 1000) * self.token_costs["input"] + (output_tokens / 1000) * self.token_costs["output"]
        if hasattr(completion_response, "_hidden_params"):
            cost = completion_response._hidden_params.get("response_cost", 0)
        try:
            import litellm
            cost = litellm.completion_cost(model=model, completion_response=completion_response)
        except ImportError:
            pass
        except Exception as e:
            try:
                import litellm
                cost = litellm.completion_cost(model=model.split('/')[-1], completion_response=completion_response)
            except Exception:
                pass
        return cost or (input_tokens / 1000) * self.token_costs["input"] + (output_tokens / 1000) * self.token_costs["output"]

    def get_summary(self) -> dict[str, Any]:
        """Get comprehensive progress summary"""
        summary = {
            "total_events": len(self.events),
            "llm_calls": len([e for e in self.events if e.event_type == "llm_call"]),
            "tool_calls": len([e for e in self.events if e.event_type == "tool_call"]),
            "total_cost": sum(e.llm_cost for e in self.events if e.llm_cost),
            "total_tokens": sum(e.llm_total_tokens for e in self.events if e.llm_total_tokens),
            "total_duration": sum(e.node_duration for e in self.events if e.node_duration),
            "nodes_visited": list(set(e.node_name for e in self.events)),
            "tools_used": list(set(e.tool_name for e in self.events if e.tool_name)),
            "models_used": list(set(e.llm_model for e in self.events if e.llm_model))
        }
        return summary

calculate_llm_cost(model, input_tokens, output_tokens, completion_response=None)

Calculate approximate LLM cost

Source code in toolboxv2/mods/isaa/base/Agent/types.py

def calculate_llm_cost(self, model: str, input_tokens: int, output_tokens: int,completion_response:Any=None) -> float:
    """Calculate approximate LLM cost"""
    cost = (input_tokens / 1000) * self.token_costs["input"] + (output_tokens / 1000) * self.token_costs["output"]
    if hasattr(completion_response, "_hidden_params"):
        cost = completion_response._hidden_params.get("response_cost", 0)
    try:
        import litellm
        cost = litellm.completion_cost(model=model, completion_response=completion_response)
    except ImportError:
        pass
    except Exception as e:
        try:
            import litellm
            cost = litellm.completion_cost(model=model.split('/')[-1], completion_response=completion_response)
        except Exception:
            pass
    return cost or (input_tokens / 1000) * self.token_costs["input"] + (output_tokens / 1000) * self.token_costs["output"]

emit_event(event) async

Emit progress event with callback and storage

Source code in toolboxv2/mods/isaa/base/Agent/types.py

async def emit_event(self, event: ProgressEvent):
    """Emit progress event with callback and storage"""
    self.events.append(event)
    event.agent_name = self.agent_name

    if self.progress_callback:
        try:
            if asyncio.iscoroutinefunction(self.progress_callback):
                await self.progress_callback(event)
            else:
                self.progress_callback(event)
        except Exception:
            import traceback
            print(traceback.format_exc())

end_timer(key)

End timing operation and return duration

Source code in toolboxv2/mods/isaa/base/Agent/types.py

def end_timer(self, key: str) -> float:
    """End timing operation and return duration"""
    if key not in self.active_timers:
        return 0.0
    duration = time.perf_counter() - self.active_timers[key]
    del self.active_timers[key]
    return duration

get_summary()

Get comprehensive progress summary

Source code in toolboxv2/mods/isaa/base/Agent/types.py

def get_summary(self) -> dict[str, Any]:
    """Get comprehensive progress summary"""
    summary = {
        "total_events": len(self.events),
        "llm_calls": len([e for e in self.events if e.event_type == "llm_call"]),
        "tool_calls": len([e for e in self.events if e.event_type == "tool_call"]),
        "total_cost": sum(e.llm_cost for e in self.events if e.llm_cost),
        "total_tokens": sum(e.llm_total_tokens for e in self.events if e.llm_total_tokens),
        "total_duration": sum(e.node_duration for e in self.events if e.node_duration),
        "nodes_visited": list(set(e.node_name for e in self.events)),
        "tools_used": list(set(e.tool_name for e in self.events if e.tool_name)),
        "models_used": list(set(e.llm_model for e in self.events if e.llm_model))
    }
    return summary

start_timer(key)

Start timing operation

Source code in toolboxv2/mods/isaa/base/Agent/types.py

def start_timer(self, key: str) -> float:
    """Start timing operation"""
    start_time = time.perf_counter()
    self.active_timers[key] = start_time
    return start_time

ResponseFinalProcessorNode

Bases: AsyncNode

Finale Verarbeitung mit Persona-System

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

@with_progress_tracking
class ResponseFinalProcessorNode(AsyncNode):
    """Finale Verarbeitung mit Persona-System"""

    async def prep_async(self, shared):
        return {
            "formatted_response": shared.get("formatted_response", {}),
            "quality_assessment": shared.get("quality_assessment", {}),
            "conversation_history": shared.get("conversation_history", []),
            "persona": shared.get("persona_config"),
            "fast_llm_model": shared.get("fast_llm_model"),
            "use_fast_response": shared.get("use_fast_response", True),
            "agent_instance": shared.get("agent_instance"),
        }

    async def exec_async(self, prep_res):
        response_data = prep_res["formatted_response"]
        raw_response = response_data.get("formatted_response", "I apologize, but I couldn't generate a response.")

        # Persona-basierte Anpassung
        if prep_res.get("persona") and LITELLM_AVAILABLE:
            final_response = await self._apply_persona_style(raw_response, prep_res)
        else:
            final_response = raw_response

        # Finale Metadaten
        processing_metadata = {
            "response_confidence": response_data.get("confidence", 0.0),
            "quality_score": prep_res.get("quality_assessment", {}).get("quality_score", 0.0),
            "processing_timestamp": datetime.now().isoformat(),
            "response_length": len(final_response),
            "persona_applied": prep_res.get("persona") is not None
        }

        return {
            "final_response": final_response,
            "metadata": processing_metadata,
            "status": "completed"
        }

    async def _apply_persona_style(self, response: str, prep_res: dict) -> str:
        """Optimized persona styling mit Konfiguration"""
        persona = prep_res["persona"]

        # Nur anwenden wenn post-processing konfiguriert
        if not persona.should_post_process():
            return response

        # Je nach Integration Level unterschiedliche Prompts
        if persona.integration_level == "light":
            style_prompt = f"Make this {persona.tone} and {persona.style}: {response}"
            max_tokens = 400
        elif persona.integration_level == "medium":
            style_prompt = f"""
    Apply {persona.name} persona (style: {persona.style}, tone: {persona.tone}) to:
    {response}

    Keep the same information, adjust presentation:"""
            max_tokens = 600
        else:  # heavy
            style_prompt = f"""
Completely transform as {persona.name}:
Style: {persona.style}, Tone: {persona.tone}
Traits: {', '.join(persona.personality_traits)}
Instructions: {persona.custom_instructions}

Original: {response}

As {persona.name}:"""
            max_tokens = 1000

        try:
            model_to_use = prep_res.get("fast_llm_model", "openrouter/anthropic/claude-3-haiku")
            agent_instance = prep_res["agent_instance"]
            if prep_res.get("use_fast_response", True):
                response = await agent_instance.a_run_llm_completion(
                    model=model_to_use,
                    messages=[{"role": "user", "content": style_prompt}],
                    temperature=0.5,
                    max_tokens=max_tokens, node_name="PersonaStylingNode", task_id="persona_styling_fast"
                )
            else:
                response = await agent_instance.a_run_llm_completion(
                    model=model_to_use,
                    messages=[{"role": "user", "content": style_prompt}],
                    temperature=0.6,
                    max_tokens=max_tokens + 200, node_name="PersonaStylingNode", task_id="persona_styling_ritch"
                )

            return response.strip()

        except Exception as e:
            wprint(f"Persona styling failed: {e}")
            return response

    async def post_async(self, shared, prep_res, exec_res):
        shared["current_response"] = exec_res["final_response"]
        shared["response_metadata"] = exec_res["metadata"]
        return "response_ready"

ResponseFormatterNode

Bases: AsyncNode

Formatiere finale Antwort für Benutzer

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

@with_progress_tracking
class ResponseFormatterNode(AsyncNode):
    """Formatiere finale Antwort für Benutzer"""

    async def prep_async(self, shared):
        return {
            "synthesized_response": shared.get("synthesized_response", {}),
            "original_query": shared.get("current_query", ""),
            "user_preferences": shared.get("user_preferences", {})
        }

    async def exec_async(self, prep_res):
        synthesis_data = prep_res["synthesized_response"]
        raw_response = synthesis_data.get("synthesized_response", "")

        if not raw_response:
            return {
                "formatted_response": "I apologize, but I was unable to generate a meaningful response to your query."}

        # Basis-Formatierung
        formatted_response = raw_response.strip()

        # Füge Metadaten hinzu falls gewünscht (für debugging/transparency)
        confidence = synthesis_data.get("confidence", 0.0)
        if confidence < 0.4:
            formatted_response += "\n\n*Note: This response has low confidence due to limited information.*"

        adaptation_note = ""
        synthesis_method = synthesis_data.get("synthesis_method", "unknown")
        if synthesis_method == "fallback":
            adaptation_note = "\n\n*Note: Response generated with limited processing capabilities.*"

        return {
            "formatted_response": formatted_response + adaptation_note,
            "confidence": confidence,
            "metadata": {
                "synthesis_method": synthesis_method,
                "response_length": len(formatted_response)
            }
        }

    async def post_async(self, shared, prep_res, exec_res):
        shared["formatted_response"] = exec_res
        return "formatted"

ResponseGenerationFlow

Bases: AsyncFlow

Intelligente Antwortgenerierung basierend auf Task-Ergebnissen

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

@with_progress_tracking
class ResponseGenerationFlow(AsyncFlow):
    """Intelligente Antwortgenerierung basierend auf Task-Ergebnissen"""

    def __init__(self, tools=None):
        # Nodes für Response-Pipeline
        self.context_aggregator = ContextAggregatorNode()
        self.result_synthesizer = ResultSynthesizerNode()
        self.response_formatter = ResponseFormatterNode()
        self.quality_checker = ResponseQualityNode()
        self.final_processor = ResponseFinalProcessorNode()

        # === RESPONSE GENERATION PIPELINE ===

        # Context Aggregation -> Synthesis
        self.context_aggregator - "context_ready" >> self.result_synthesizer
        self.context_aggregator - "no_context" >> self.response_formatter  # Fallback

        # Synthesis -> Formatting
        self.result_synthesizer - "synthesized" >> self.response_formatter
        self.result_synthesizer - "synthesis_failed" >> self.response_formatter

        # Formatting -> Quality Check
        self.response_formatter - "formatted" >> self.quality_checker
        self.response_formatter - "format_failed" >> self.final_processor  # Skip quality check

        # Quality Check -> Final Processing oder Retry
        self.quality_checker - "quality_good" >> self.final_processor
        self.quality_checker - "quality_poor" >> self.result_synthesizer  # Retry synthesis
        self.quality_checker - "quality_acceptable" >> self.final_processor

        super().__init__(start=self.context_aggregator)

ResponseQualityNode

Bases: AsyncNode

Prüfe Qualität der generierten Antwort

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

@with_progress_tracking
class ResponseQualityNode(AsyncNode):
    """Prüfe Qualität der generierten Antwort"""

    async def prep_async(self, shared):
        return {
            "formatted_response": shared.get("formatted_response", {}),
            "original_query": shared.get("current_query", ""),
            "format_config": self._get_format_config(shared),
            "fast_llm_model": shared.get("fast_llm_model"),
            "persona_config": shared.get("persona_config"),
            "agent_instance": shared.get("agent_instance"),
        }

    def _get_format_config(self, shared) -> FormatConfig | None:
        """Extrahiere Format-Konfiguration"""
        persona = shared.get("persona_config")
        if persona and hasattr(persona, 'format_config'):
            return persona.format_config
        return None

    async def exec_async(self, prep_res):
        response_data = prep_res["formatted_response"]
        response_text = response_data.get("formatted_response", "")
        original_query = prep_res["original_query"]
        format_config = prep_res["format_config"]

        # Basis-Qualitätsprüfung
        base_quality = self._heuristic_quality_check(response_text, original_query)

        # Format-spezifische Bewertung
        format_quality = await self._evaluate_format_adherence(response_text, format_config)

        # Längen-spezifische Bewertung
        length_quality = self._evaluate_length_adherence(response_text, format_config)

        # LLM-basierte Gesamtbewertung
        llm_quality = 0.5
        if LITELLM_AVAILABLE and len(response_text) > 500:
            llm_quality = await self._llm_format_quality_check(
                response_text, original_query, format_config, prep_res
            )

        # Gewichtete Gesamtbewertung
        total_quality = (
            base_quality * 0.3 +
            format_quality * 0.3 +
            length_quality * 0.2 +
            llm_quality * 0.2
        )

        quality_details = {
            "total_score": total_quality,
            "base_quality": base_quality,
            "format_adherence": format_quality,
            "length_adherence": length_quality,
            "llm_assessment": llm_quality,
            "format_config_used": format_config is not None
        }

        return {
            "quality_score": total_quality,
            "quality_assessment": self._score_to_assessment(total_quality),
            "quality_details": quality_details,
            "suggestions": self._generate_format_quality_suggestions(
                total_quality, response_text, format_config, quality_details
            )
        }

    async def _evaluate_format_adherence(self, response: str, format_config: FormatConfig | None) -> float:
        """Bewerte Format-Einhaltung"""
        if not format_config:
            return 0.8  # Neutral wenn kein Format vorgegeben

        format_type = format_config.response_format
        score = 0.5

        # Format-spezifische Checks
        if format_type == ResponseFormat.WITH_TABLES:
            if '|' in response or 'Table:' in response or '| ' in response:
                score += 0.4

        elif format_type == ResponseFormat.WITH_BULLET_POINTS:
            bullet_count = response.count('•') + response.count('-') + response.count('*')
            if bullet_count >= 2:
                score += 0.4
            elif bullet_count >= 1:
                score += 0.2

        elif format_type == ResponseFormat.WITH_LISTS:
            list_patterns = ['1.', '2.', '3.', 'a)', 'b)', 'c)']
            list_score = sum(1 for pattern in list_patterns if pattern in response)
            score += min(0.4, list_score * 0.1)

        elif format_type == ResponseFormat.MD_TEXT:
            md_elements = ['#', '**', '*', '`', '```', '[', ']', '(', ')']
            md_score = sum(1 for element in md_elements if element in response)
            score += min(0.4, md_score * 0.05)

        elif format_type == ResponseFormat.YAML_TEXT:
            if response.strip().startswith(('```yaml', '---')) or ': ' in response:
                score += 0.4

        elif format_type == ResponseFormat.JSON_TEXT:
            if response.strip().startswith(('{', '[')):
                try:
                    json.loads(response)
                    score += 0.4
                except:
                    score += 0.1  # Partial credit for JSON-like structure

        elif format_type == ResponseFormat.TEXT_ONLY:
            # Penalize if formatting elements are present
            format_elements = ['#', '*', '|', '```', '1.', '•', '-']
            format_count = sum(1 for element in format_elements if element in response)
            score += max(0.1, 0.5 - format_count * 0.05)

        elif format_type == ResponseFormat.PSEUDO_CODE:
            code_indicators = ['if ', 'for ', 'while ', 'def ', 'return ', 'function', 'BEGIN', 'END']
            code_score = sum(1 for indicator in code_indicators if indicator in response)
            score += min(0.4, code_score * 0.1)

        return max(0.0, min(1.0, score))

    def _evaluate_length_adherence(self, response: str, format_config: FormatConfig | None) -> float:
        """Bewerte Längen-Einhaltung"""
        if not format_config:
            return 0.8

        word_count = len(response.split())
        min_words, max_words = format_config.get_expected_word_range()

        if min_words <= word_count <= max_words:
            return 1.0
        elif word_count < min_words:
            # Zu kurz - sanfte Bestrafung
            ratio = word_count / min_words
            return max(0.3, ratio * 0.8)
        else:  # word_count > max_words
            # Zu lang - weniger Bestrafung als zu kurz
            excess_ratio = (word_count - max_words) / max_words
            return max(0.4, 1.0 - excess_ratio * 0.3)

    async def _llm_format_quality_check(
        self,
        response: str,
        query: str,
        format_config: FormatConfig | None,
        prep_res: dict
    ) -> float:
        """LLM-basierte Format- und Qualitätsbewertung"""
        if not format_config:
            return await self._standard_llm_quality_check(response, query, prep_res)

        format_desc = format_config.get_format_instructions()
        length_desc = format_config.get_length_instructions()

        prompt = f"""
Bewerte diese Antwort auf einer Skala von 0.0 bis 1.0 basierend auf Format-Einhaltung und Qualität:

Benutzer-Anfrage: {query}

Antwort: {response}

Erwartetes Format: {format_desc}
Erwartete Länge: {length_desc}

Bewertungskriterien:
1. Format-Einhaltung (40%): Entspricht die Antwort dem geforderten Format?
2. Längen-Angemessenheit (25%): Ist die Länge angemessen?
3. Inhaltliche Qualität (25%): Beantwortet die Anfrage vollständig?
4. Lesbarkeit und Struktur (10%): Ist die Antwort gut strukturiert?

Antworte nur mit einer Zahl zwischen 0.0 und 1.0:"""

        try:
            model_to_use = prep_res.get("fast_llm_model", "openrouter/anthropic/claude-3-haiku")
            agent_instance = prep_res["agent_instance"]
            score_text = (await agent_instance.a_run_llm_completion(
                model=model_to_use,
                messages=[{"role": "user", "content": prompt}],
                temperature=0.1,
                max_tokens=10,
                node_name="QualityAssessmentNode", task_id="format_quality_assessment"
            )).strip()

            return float(score_text)

        except Exception as e:
            wprint(f"LLM format quality check failed: {e}")
            return 0.6  # Neutral fallback

    def _generate_format_quality_suggestions(
        self,
        score: float,
        response: str,
        format_config: FormatConfig | None,
        quality_details: dict
    ) -> list[str]:
        """Generiere Format-spezifische Verbesserungsvorschläge"""
        suggestions = []

        if not format_config:
            return ["Consider defining a specific response format for better consistency"]

        # Format-spezifische Vorschläge
        if quality_details["format_adherence"] < 0.6:
            format_type = format_config.response_format

            if format_type == ResponseFormat.WITH_TABLES:
                suggestions.append("Add tables using markdown format (| Column | Column |)")
            elif format_type == ResponseFormat.WITH_BULLET_POINTS:
                suggestions.append("Use bullet points (•, -, *) to structure information")
            elif format_type == ResponseFormat.MD_TEXT:
                suggestions.append("Use markdown formatting (headers, bold, code blocks)")
            elif format_type == ResponseFormat.YAML_TEXT:
                suggestions.append("Format response as valid YAML structure")
            elif format_type == ResponseFormat.JSON_TEXT:
                suggestions.append("Format response as valid JSON")

        # Längen-spezifische Vorschläge
        if quality_details["length_adherence"] < 0.6:
            word_count = len(response.split())
            min_words, max_words = format_config.get_expected_word_range()

            if word_count < min_words:
                suggestions.append(f"Response too short ({word_count} words). Aim for {min_words}-{max_words} words")
            else:
                suggestions.append(f"Response too long ({word_count} words). Aim for {min_words}-{max_words} words")

        # Qualitäts-spezifische Vorschläge
        if score < 0.5:
            suggestions.append("Overall quality needs improvement - consider regenerating")
        elif score < 0.7:
            suggestions.append("Good response but could be enhanced with better format adherence")

        return suggestions

    async def _standard_llm_quality_check(self, response: str, query: str, prep_res: dict) -> float:
        """Standard LLM-Qualitätsprüfung ohne Format-Fokus"""
        # Bestehende Implementierung beibehalten
        return await self._llm_quality_check(response, query, prep_res)

    def _heuristic_quality_check(self, response: str, query: str) -> float:
        """Heuristische Qualitätsprüfung"""
        score = 0.5  # Base score

        # Length check
        if len(response) < 50:
            score -= 0.3
        elif len(response) > 100:
            score += 0.2

        # Query term coverage
        query_terms = set(query.lower().split())
        response_terms = set(response.lower().split())
        coverage = len(query_terms.intersection(response_terms)) / max(len(query_terms), 1)
        score += coverage * 0.3

        # Structure indicators
        if any(indicator in response for indicator in [":", "-", "1.", "•"]):
            score += 0.1  # Structured response bonus

        return max(0.0, min(1.0, score))

    async def _llm_quality_check(self, response: str, query: str, prep_res: dict) -> float:
        """LLM-basierte Qualitätsprüfung"""
        try:
            prompt = f"""
Rate the quality of this response to the user's query on a scale of 0.0 to 1.0.

User Query: {query}

Response: {response}

Consider:
- Relevance to the query
- Completeness of information
- Clarity and readability
- Accuracy (if verifiable)

Respond with just a number between 0.0 and 1.0:"""

            model_to_use = prep_res.get("fast_llm_model", "openrouter/anthropic/claude-3-haiku")
            agent_instance = prep_res["agent_instance"]
            score_text = (await agent_instance.a_run_llm_completion(
                model=model_to_use,
                messages=[{"role": "user", "content": prompt}],
                temperature=0.1,
                max_tokens=10,
                node_name="QualityAssessmentNode", task_id="quality_assessment"
            )).strip()

            return float(score_text)

        except:
            return 0.5  # Fallback score

    def _score_to_assessment(self, score: float) -> str:
        if score >= 0.8:
            return "quality_good"
        elif score >= 0.5:
            return "quality_acceptable"
        else:
            return "quality_poor"

    async def post_async(self, shared, prep_res, exec_res):
        shared["quality_assessment"] = exec_res
        return exec_res["quality_assessment"]

ResultSynthesizerNode

Bases: AsyncNode

Synthetisiere finale Antwort aus allen Ergebnissen

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

@with_progress_tracking
class ResultSynthesizerNode(AsyncNode):
    """Synthetisiere finale Antwort aus allen Ergebnissen"""

    async def prep_async(self, shared):
        return {
            "aggregated_context": shared.get("aggregated_context", {}),
            "fast_llm_model": shared.get("fast_llm_model"),
            "complex_llm_model": shared.get("complex_llm_model"),
            "agent_instance": shared.get("agent_instance")
        }

    async def exec_async(self, prep_res):
        if not LITELLM_AVAILABLE:
            return await self._fallback_synthesis(prep_res)

        context = prep_res["aggregated_context"]
        persona = (prep_res['agent_instance'].amd.persona.to_system_prompt_addition() if not prep_res['agent_instance'].amd.persona.should_post_process() else '') if prep_res['agent_instance'].amd.persona else None
        prompt = f"""
Du bist ein Experte für Informationssynthese. Erstelle eine umfassende, hilfreiche Antwort basierend auf den gesammelten Ergebnissen.

## Ursprüngliche Anfrage
{context.get('original_query', '')}

## Erfolgreiche Ergebnisse
{self._format_successful_results(context.get('successful_results', {}))}

## Wichtige Entdeckungen
{self._format_key_discoveries(context.get('key_discoveries', []))}

## Plan-Adaptationen
{context.get('adaptation_summary', 'No adaptations were needed.')}

## Fehlgeschlagene Versuche
{self._format_failed_attempts(context.get('failed_attempts', {}))}

{persona}

## Anweisungen
1. Gib eine direkte, hilfreiche Antwort auf die ursprüngliche Anfrage
2. Integriere alle relevanten gefundenen Informationen
3. Erkläre kurz den Prozess, falls Adaptationen nötig waren
4. Sei ehrlich über Limitationen oder fehlende Informationen
5. Strukturiere die Antwort logisch und lesbar

Erstelle eine finale Antwort:"""

        try:
            # Verwende complex model für finale Synthesis
            model_to_use = prep_res.get("complex_llm_model", "openrouter/openai/gpt-4o")
            agent_instance = prep_res["agent_instance"]
            synthesized_response = await agent_instance.a_run_llm_completion(
                model=model_to_use,
                messages=[{"role": "user", "content": prompt}],
                temperature=0.3,
                max_tokens=1500,
                node_name="ResultSynthesizerNode", task_id="response_synthesis"
            )

            return {
                "synthesized_response": synthesized_response,
                "synthesis_method": "llm",
                "model_used": model_to_use,
                "confidence": self._estimate_synthesis_confidence(context)
            }

        except Exception as e:
            eprint(f"LLM synthesis failed: {e}")
            return await self._fallback_synthesis(prep_res)

    def _format_successful_results(self, results: dict) -> str:
        formatted = []
        for _task_id, result_info in results.items():
            formatted.append(f"- {result_info['task_description']}: {str(result_info['result'])[:20000]}...")
        return "\n".join(formatted) if formatted else "No successful results to report."

    def _format_key_discoveries(self, discoveries: list) -> str:
        formatted = []
        for discovery in discoveries:
            confidence = discovery.get('confidence', 0.0)
            formatted.append(f"- {discovery['discovery']} (Confidence: {confidence:.2f})")
        return "\n".join(formatted) if formatted else "No key discoveries."

    def _format_failed_attempts(self, failed: dict) -> str:
        if not failed:
            return "No significant failures."
        formatted = [f"- {info['description']}: {info['error']}" for info in failed.values()]
        return "\n".join(formatted)

    async def _fallback_synthesis(self, prep_res) -> dict:
        """Fallback synthesis ohne LLM"""
        context = prep_res["aggregated_context"]

        # Einfache Template-basierte Synthese
        response_parts = []

        if context.get("key_discoveries"):
            response_parts.append("Based on my analysis, I found:")
            for discovery in context["key_discoveries"][:3]:  # Top 3
                response_parts.append(f"- {discovery['discovery']}")

        if context.get("successful_results"):
            response_parts.append("\nDetailed results:")
            for _task_id, result in list(context["successful_results"].items())[:2]:  # Top 2
                response_parts.append(f"- {result['task_description']}: {str(result['result'])[:150]}")

        if context.get("adaptation_summary"):
            response_parts.append(f"\n{context['adaptation_summary']}")

        fallback_response = "\n".join(
            response_parts) if response_parts else "I was unable to complete the requested task effectively."

        return {
            "synthesized_response": fallback_response,
            "synthesis_method": "fallback",
            "confidence": 0.3
        }

    def _estimate_synthesis_confidence(self, context: dict) -> float:
        """Schätze Confidence der Synthese"""
        confidence = 0.5  # Base confidence

        # Boost für erfolgreiche Ergebnisse
        successful_count = len(context.get("successful_results", {}))
        confidence += min(successful_count * 0.15, 0.3)

        # Boost für key discoveries mit hoher confidence
        for discovery in context.get("key_discoveries", []):
            discovery_conf = discovery.get("confidence", 0.0)
            confidence += discovery_conf * 0.1

        # Penalty für viele fehlgeschlagene Versuche
        failed_count = len(context.get("failed_attempts", {}))
        confidence -= min(failed_count * 0.1, 0.2)

        return max(0.1, min(1.0, confidence))

    async def post_async(self, shared, prep_res, exec_res):
        shared["synthesized_response"] = exec_res
        if exec_res.get("synthesized_response"):
            return "synthesized"
        else:
            return "synthesis_failed"

StateSyncNode

Bases: AsyncNode

Synchronize state between world model and shared store

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

@with_progress_tracking
class StateSyncNode(AsyncNode):
    """Synchronize state between world model and shared store"""
    async def prep_async(self, shared):
        world_model = shared.get("world_model", {})
        session_data = shared.get("session_data", {})
        tasks = shared.get("tasks", {})
        system_status = shared.get("system_status", "idle")

        return {
            "world_model": world_model,
            "session_data": session_data,
            "tasks": tasks,
            "system_status": system_status,
            "sync_timestamp": datetime.now().isoformat()
        }

    async def exec_async(self, prep_res):
        # Perform intelligent state synchronization
        sync_result = {
            "world_model_updates": {},
            "session_updates": {},
            "task_updates": {},
            "conflicts_resolved": [],
            "sync_successful": True
        }

        # Update world model with new information
        if "current_response" in prep_res:
            # Extract learnable facts from responses
            extracted_facts = self._extract_facts(prep_res.get("current_response", ""))
            sync_result["world_model_updates"].update(extracted_facts)

        # Sync task states
        for task_id, task in prep_res["tasks"].items():
            if task.status == "completed" and task.result:
                # Store task results in world model
                fact_key = f"task_{task_id}_result"
                sync_result["world_model_updates"][fact_key] = task.result

        return sync_result

    def _extract_facts(self, text: str) -> dict[str, Any]:
        """Extract learnable facts from text"""
        facts = {}
        lines = text.split('\n')

        for line in lines:
            line = line.strip()
            # Look for definitive statements
            if ' is ' in line and not line.startswith('I ') and not '?' in line:
                parts = line.split(' is ', 1)
                if len(parts) == 2:
                    subject = parts[0].strip().lower()
                    predicate = parts[1].strip().rstrip('.')
                    if len(subject.split()) <= 3:  # Keep subjects simple
                        facts[subject] = predicate

        return facts

    async def post_async(self, shared, prep_res, exec_res):
        # Apply the synchronization results
        if exec_res["sync_successful"]:
            shared["world_model"].update(exec_res["world_model_updates"])
            shared["session_data"].update(exec_res["session_updates"])
            shared["last_sync"] = datetime.now()
            return "sync_complete"
        else:
            wprint("State synchronization failed")
            return "sync_failed"

Task dataclass

Source code in toolboxv2/mods/isaa/base/Agent/types.py

@dataclass
class Task:
    id: str
    type: str
    description: str
    status: str = "pending"  # pending, running, completed, failed, paused
    priority: int = 1
    dependencies: list[str] = field(default_factory=list)
    subtasks: list[str] = field(default_factory=list)
    result: Any = None
    error: str = None
    created_at: datetime = field(default_factory=datetime.now)
    started_at: datetime  = None
    completed_at: datetime  = None
    metadata: dict[str, Any] = field(default_factory=dict)
    retry_count: int = 0
    max_retries: int = 3
    critical: bool = False

    task_identification_attr: bool = True


    def __post_init__(self):
        """Ensure all mutable defaults are properly initialized"""
        if self.metadata is None:
            self.metadata = {}
        if self.dependencies is None:
            self.dependencies = []
        if self.subtasks is None:
            self.subtasks = []

    def __getitem__(self, key):
        return getattr(self, key)

    def __setitem__(self, key, value):
        setattr(self, key, value)

__post_init__()

Ensure all mutable defaults are properly initialized

Source code in toolboxv2/mods/isaa/base/Agent/types.py

def __post_init__(self):
    """Ensure all mutable defaults are properly initialized"""
    if self.metadata is None:
        self.metadata = {}
    if self.dependencies is None:
        self.dependencies = []
    if self.subtasks is None:
        self.subtasks = []

TaskExecutorNode

Bases: AsyncNode

Vollständige Task-Ausführung als unabhängige Node mit LLM-unterstützter Planung

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

@with_progress_tracking
class TaskExecutorNode(AsyncNode):
    """Vollständige Task-Ausführung als unabhängige Node mit LLM-unterstützter Planung"""

    def __init__(self, max_parallel: int = 3, **kwargs):
        super().__init__(**kwargs)
        self.max_parallel = max_parallel
        self.results_store = {}  # Für {{ }} Referenzen
        self.execution_history = []  # Für LLM-basierte Optimierung
        self.agent_instance = None  # Wird gesetzt vom FlowAgent
        self.variable_manager = None
        self.fast_llm_model = None
        self.complex_llm_model = None
        self.progress_tracker = None

    async def prep_async(self, shared):
        """Enhanced preparation with unified variable system"""
        current_plan = shared.get("current_plan")
        tasks = shared.get("tasks", {})

        # Get unified variable manager
        self.variable_manager = shared.get("variable_manager")
        self.progress_tracker = shared.get("progress_tracker")
        if not self.variable_manager:
            self.variable_manager = VariableManager(shared.get("world_model", {}), shared)

        # Register all necessary scopes
        self.variable_manager.set_results_store(self.results_store)
        self.variable_manager.set_tasks_store(tasks)
        self.variable_manager.register_scope('user', shared.get('user_context', {}))
        self.variable_manager.register_scope('system', {
            'timestamp': datetime.now().isoformat(),
            'agent_name': shared.get('agent_instance', {}).amd.name if shared.get('agent_instance') else 'unknown'
        })

        # Stelle sicher, dass Agent-Referenz verfügbar ist
        if not self.agent_instance:
            self.agent_instance = shared.get("agent_instance")

        if not current_plan:
            return {"error": "No active plan", "tasks": tasks}

        # Rest of existing prep_async logic...
        ready_tasks = self._find_ready_tasks(current_plan, tasks)
        blocked_tasks = self._find_blocked_tasks(current_plan, tasks)

        execution_plan = await self._create_intelligent_execution_plan(
            ready_tasks, blocked_tasks, current_plan, shared
        )
        self.complex_llm_model = shared.get("complex_llm_model")
        self.fast_llm_model = shared.get("fast_llm_model")

        return {
            "plan": current_plan,
            "ready_tasks": ready_tasks,
            "blocked_tasks": blocked_tasks,
            "all_tasks": tasks,
            "execution_plan": execution_plan,
            "fast_llm_model": self.fast_llm_model,
            "complex_llm_model": self.complex_llm_model,
            "available_tools": shared.get("available_tools", []),
            "world_model": shared.get("world_model", {}),
            "results": self.results_store,
            "variable_manager": self.variable_manager,
            "progress_tracker": self.progress_tracker ,
        }

    def _find_ready_tasks(self, plan: TaskPlan, all_tasks: dict[str, Task]) -> list[Task]:
        """Finde Tasks die zur Ausführung bereit sind"""
        ready = []
        for task in plan.tasks:
            if task.status == "pending" and self._dependencies_satisfied(task, all_tasks):
                ready.append(task)
        return ready

    def _find_blocked_tasks(self, plan: TaskPlan, all_tasks: dict[str, Task]) -> list[Task]:
        """Finde blockierte Tasks für Analyse"""
        blocked = []
        for task in plan.tasks:
            if task.status == "pending" and not self._dependencies_satisfied(task, all_tasks):
                blocked.append(task)
        return blocked

    def _dependencies_satisfied(self, task: Task, all_tasks: dict[str, Task]) -> bool:
        """Prüfe ob alle Dependencies erfüllt sind"""
        for dep_id in task.dependencies:
            if dep_id in all_tasks:
                dep_task = all_tasks[dep_id]
                if dep_task.status not in ["completed"]:
                    return False
            else:
                # Dependency existiert nicht - könnte Problem sein
                wprint(f"Task {task.id} has missing dependency: {dep_id}")
                return False
        return True

    async def _create_intelligent_execution_plan(
        self,
        ready_tasks: list[Task],
        blocked_tasks: list[Task],
        plan: TaskPlan,
        shared: dict
    ) -> dict[str, Any]:
        """LLM-unterstützte intelligente Ausführungsplanung"""

        if not ready_tasks:
            return {
                "strategy": "waiting",
                "reason": "No ready tasks",
                "blocked_count": len(blocked_tasks),
                "recommendations": []
            }

        # Einfache Planung für wenige Tasks
        if len(ready_tasks) <= 2 and not LITELLM_AVAILABLE:
            return self._create_simple_execution_plan(ready_tasks, plan)

        # LLM-basierte intelligente Planung
        return await self._llm_execution_planning(ready_tasks, blocked_tasks, plan, shared)

    def _create_simple_execution_plan(self, ready_tasks: list[Task], plan: TaskPlan) -> dict[str, Any]:
        """Einfache heuristische Ausführungsplanung"""

        # Prioritäts-basierte Sortierung
        sorted_tasks = sorted(ready_tasks, key=lambda t: (t.priority, t.created_at))

        # Parallelisierbare Tasks identifizieren
        parallel_groups = []
        current_group = []

        for task in sorted_tasks:
            # ToolTasks können oft parallel laufen
            if isinstance(task, ToolTask) and len(current_group) < self.max_parallel:
                current_group.append(task)
            else:
                if current_group:
                    parallel_groups.append(current_group)
                    current_group = []
                current_group.append(task)

        if current_group:
            parallel_groups.append(current_group)

        strategy = "parallel" if len(parallel_groups) > 1 or len(parallel_groups[0]) > 1 else "sequential"

        return {
            "strategy": strategy,
            "execution_groups": parallel_groups,
            "total_groups": len(parallel_groups),
            "reasoning": "Simple heuristic: priority-based with tool parallelization",
            "estimated_duration": self._estimate_duration(sorted_tasks)
        }

    async def _llm_execution_planning(
        self,
        ready_tasks: list[Task],
        blocked_tasks: list[Task],
        plan: TaskPlan,
        shared: dict
    ) -> dict[str, Any]:
        """Erweiterte LLM-basierte Ausführungsplanung"""

        try:
            # Erstelle detaillierte Task-Analyse für LLM
            task_analysis = self._analyze_tasks_for_llm(ready_tasks, blocked_tasks)
            execution_context = self._build_execution_context(shared)

            prompt = f"""
Du bist ein Experte für Task-Ausführungsplanung. Analysiere die verfügbaren Tasks und erstelle einen optimalen Ausführungsplan.

## Verfügbare Tasks zur Ausführung
{task_analysis['ready_tasks_summary']}

## Blockierte Tasks (zur Information)
{task_analysis['blocked_tasks_summary']}

## Ausführungskontext
- Max parallele Tasks: {self.max_parallel}
- Plan-Strategie: {plan.execution_strategy}
- Verfügbare Tools: {', '.join(shared.get('available_tools', []))}
- Bisherige Ergebnisse: {len(self.results_store)} Tasks abgeschlossen
- Execution History: {len(self.execution_history)} vorherige Zyklen

## Bisherige Performance
{execution_context}

## Aufgabe
Erstelle einen optimierten Ausführungsplan. Berücksichtige:
1. Task-Abhängigkeiten und Prioritäten
2. Parallelisierungsmöglichkeiten
3. Resource-Optimierung (Tools, LLM-Aufrufe)
4. Fehlerwahrscheinlichkeit und Retry-Strategien
5. Dynamische Argument-Auflösung zwischen Tasks

Antworte mit YAML:

```yaml
strategy: "parallel"  # "parallel" | "sequential" | "hybrid"
execution_groups:
  - group_id: 1
    tasks: ["task_1", "task_2"]  # Task IDs
    execution_mode: "parallel"
    priority: "high"
    estimated_duration: 30  # seconds
    risk_level: "low"  # low | medium | high
    dependencies_resolved: true
  - group_id: 2
    tasks: ["task_3"]
    execution_mode: "sequential"
    priority: "medium"
    estimated_duration: 15
    depends_on_groups: [1]
reasoning: "Detailed explanation of the execution strategy"
optimization_suggestions:
  - "Specific optimization 1"
  - "Specific optimization 2"
risk_mitigation:
  - risk: "Tool timeout"
    mitigation: "Use shorter timeout for parallel calls"
  - risk: "Argument resolution failure"
    mitigation: "Validate references before execution"
total_estimated_duration: 45
confidence: 0.85
```"""

            model_to_use = shared.get("complex_llm_model", "openrouter/openai/gpt-4o")

            content = await self.agent_instance.a_run_llm_completion(
                model=model_to_use,
                messages=[{"role": "user", "content": prompt}],
                temperature=0.3,
                max_tokens=2000,
                node_name="TaskExecutorNode", task_id="llm_execution_planning"
            )

            yaml_match = re.search(r"```yaml\s*(.*?)\s*```", content, re.DOTALL)
            yaml_str = yaml_match.group(1) if yaml_match else content.strip()

            execution_plan = yaml.safe_load(yaml_str)

            # Validiere und erweitere den Plan
            validated_plan = self._validate_execution_plan(execution_plan, ready_tasks)

            rprint(
                f"LLM execution plan created: {validated_plan.get('strategy')} with {len(validated_plan.get('execution_groups', []))} groups")
            return validated_plan

        except Exception as e:
            eprint(f"LLM execution planning failed: {e}")
            return self._create_simple_execution_plan(ready_tasks, plan)

    def _analyze_tasks_for_llm(self, ready_tasks: list[Task], blocked_tasks: list[Task]) -> dict[str, str]:
        """Analysiere Tasks für LLM-Prompt"""

        ready_summary = []
        for task in ready_tasks:
            task_info = f"- {task.id} ({task.type}): {task.description}"
            if hasattr(task, 'priority'):
                task_info += f" [Priority: {task.priority}]"
            if isinstance(task, ToolTask):
                task_info += f" [Tool: {task.tool_name}]"
                if task.arguments:
                    # Zeige dynamische Referenzen
                    dynamic_refs = [arg for arg in task.arguments.values() if isinstance(arg, str) and "{{" in arg]
                    if dynamic_refs:
                        task_info += f" [Dynamic refs: {len(dynamic_refs)}]"
            ready_summary.append(task_info)

        blocked_summary = []
        for task in blocked_tasks:
            deps = ", ".join(task.dependencies) if task.dependencies else "None"
            blocked_summary.append(f"- {task.id}: waiting for [{deps}]")

        return {
            "ready_tasks_summary": "\n".join(ready_summary) or "No ready tasks",
            "blocked_tasks_summary": "\n".join(blocked_summary) or "No blocked tasks"
        }

    def _build_execution_context(self, shared: dict) -> str:
        """Baue Kontext für LLM-Planung"""
        context_parts = []

        # Performance der letzten Executions
        if self.execution_history:
            recent = self.execution_history[-3:]  # Last 3 executions
            avg_duration = sum(h.get("duration", 0) for h in recent) / len(recent)
            success_rate = sum(1 for h in recent if h.get("success", False)) / len(recent)
            context_parts.append(f"Recent performance: {avg_duration:.1f}s avg, {success_rate:.1%} success rate")

        # Resource utilization
        if self.results_store:
            tool_usage = {}
            for task_result in self.results_store.values():
                metadata = task_result.get("metadata", {})
                task_type = metadata.get("task_type", "unknown")
                tool_usage[task_type] = tool_usage.get(task_type, 0) + 1
            context_parts.append(f"Resource usage: {tool_usage}")

        return "\n".join(context_parts) if context_parts else "No previous execution history"

    def _validate_execution_plan(self, plan: dict, ready_tasks: list[Task]) -> dict:
        """Validiere und korrigiere LLM-generierten Ausführungsplan"""

        # Standard-Werte setzen
        validated = {
            "strategy": plan.get("strategy", "sequential"),
            "execution_groups": [],
            "reasoning": plan.get("reasoning", "LLM-generated plan"),
            "total_estimated_duration": plan.get("total_estimated_duration", 60),
            "confidence": min(1.0, max(0.0, plan.get("confidence", 0.5)))
        }

        # Validiere execution groups
        task_ids_available = [t.id for t in ready_tasks]

        for group_data in plan.get("execution_groups", []):
            group_tasks = group_data.get("tasks", [])
            # Filtere nur verfügbare Tasks
            valid_tasks = [tid for tid in group_tasks if tid in task_ids_available]

            if valid_tasks:
                validated["execution_groups"].append({
                    "group_id": group_data.get("group_id", len(validated["execution_groups"]) + 1),
                    "tasks": valid_tasks,
                    "execution_mode": group_data.get("execution_mode", "sequential"),
                    "priority": group_data.get("priority", "medium"),
                    "estimated_duration": group_data.get("estimated_duration", 30),
                    "risk_level": group_data.get("risk_level", "medium")
                })

        # Falls keine validen Groups, erstelle Fallback
        if not validated["execution_groups"]:
            validated["execution_groups"] = [{
                "group_id": 1,
                "tasks": task_ids_available[:self.max_parallel],
                "execution_mode": "parallel",
                "priority": "high"
            }]

        return validated

    def _estimate_duration(self, tasks: list[Task]) -> int:
        """Schätze Ausführungsdauer in Sekunden"""
        duration = 0
        for task in tasks:
            if isinstance(task, ToolTask):
                duration += 10  # Tool calls meist schneller
            elif isinstance(task, LLMTask):
                duration += 20  # LLM calls brauchen länger
            else:
                duration += 15  # Standard
        return duration

    async def exec_async(self, prep_res):
        """Hauptausführungslogik mit intelligentem Routing"""

        if "error" in prep_res:
            return {"error": prep_res["error"]}

        execution_plan = prep_res["execution_plan"]

        if execution_plan["strategy"] == "waiting":
            return {
                "status": "waiting",
                "message": execution_plan["reason"],
                "blocked_count": execution_plan.get("blocked_count", 0)
            }

        # Starte Ausführung basierend auf Plan
        execution_start = datetime.now()

        try:
            if execution_plan["strategy"] == "parallel":
                results = await self._execute_parallel_plan(execution_plan, prep_res)
            elif execution_plan["strategy"] == "sequential":
                results = await self._execute_sequential_plan(execution_plan, prep_res)
            else:  # hybrid
                results = await self._execute_hybrid_plan(execution_plan, prep_res)

            execution_duration = (datetime.now() - execution_start).total_seconds()

            # Speichere Execution-History für LLM-Optimierung
            self.execution_history.append({
                "timestamp": execution_start.isoformat(),
                "strategy": execution_plan["strategy"],
                "duration": execution_duration,
                "tasks_executed": len(results),
                "success": all(r.get("status") == "completed" for r in results),
                "plan_confidence": execution_plan.get("confidence", 0.5)
            })

            # Behalte nur letzte 10 Executions
            if len(self.execution_history) > 10:
                self.execution_history = self.execution_history[-10:]

            return {
                "status": "executed",
                "results": results,
                "execution_duration": execution_duration,
                "strategy_used": execution_plan["strategy"],
                "completed_tasks": len([r for r in results if r.get("status") == "completed"]),
                "failed_tasks": len([r for r in results if r.get("status") == "failed"])
            }

        except Exception as e:
            eprint(f"Execution plan failed: {e}")
            return {
                "status": "execution_failed",
                "error": str(e),
                "results": []
            }

    async def _execute_parallel_plan(self, plan: dict, prep_res: dict) -> list[dict]:
        """Führe Plan mit parallelen Gruppen aus"""
        all_results = []

        for group in plan["execution_groups"]:
            group_tasks = self._get_tasks_by_ids(group["tasks"], prep_res)

            if group.get("execution_mode") == "parallel":
                # Parallele Ausführung innerhalb der Gruppe
                batch_results = await self._execute_parallel_batch(group_tasks)
            else:
                # Sequenzielle Ausführung innerhalb der Gruppe
                batch_results = await self._execute_sequential_batch(group_tasks)

            all_results.extend(batch_results)

            # Prüfe ob kritische Tasks fehlgeschlagen sind
            critical_failures = [
                r for r in batch_results
                if r.get("status") == "failed" and self._is_critical_task(r.get("task_id"), prep_res)
            ]

            if critical_failures:
                eprint(f"Critical task failures in group {group['group_id']}, stopping execution")
                break

        return all_results

    async def _execute_sequential_plan(self, plan: dict, prep_res: dict) -> list[dict]:
        """Führe Plan sequenziell aus"""
        all_results = []

        for group in plan["execution_groups"]:
            group_tasks = self._get_tasks_by_ids(group["tasks"], prep_res)
            batch_results = await self._execute_sequential_batch(group_tasks)
            all_results.extend(batch_results)

            # Stoppe bei kritischen Fehlern
            critical_failures = [
                r for r in batch_results
                if r.get("status") == "failed" and self._is_critical_task(r.get("task_id"), prep_res)
            ]

            if critical_failures:
                break

        return all_results

    async def _execute_hybrid_plan(self, plan: dict, prep_res: dict) -> list[dict]:
        """Hybride Ausführung - Groups parallel, innerhalb je nach Mode"""

        # Führe Gruppen parallel aus (wenn möglich)
        group_tasks_list = []
        for group in plan["execution_groups"]:
            group_tasks = self._get_tasks_by_ids(group["tasks"], prep_res)
            group_tasks_list.append((group, group_tasks))

        # Führe bis zu max_parallel Gruppen parallel aus
        batch_size = min(len(group_tasks_list), self.max_parallel)
        all_results = []

        for i in range(0, len(group_tasks_list), batch_size):
            batch = group_tasks_list[i:i + batch_size]

            # Erstelle Coroutines für jede Gruppe
            group_coroutines = []
            for group, tasks in batch:
                if group.get("execution_mode") == "parallel":
                    coro = self._execute_parallel_batch(tasks)
                else:
                    coro = self._execute_sequential_batch(tasks)
                group_coroutines.append(coro)

            # Führe Gruppen-Batch parallel aus
            batch_results = await asyncio.gather(*group_coroutines, return_exceptions=True)

            # Flache Liste der Ergebnisse
            for result_group in batch_results:
                if isinstance(result_group, Exception):
                    eprint(f"Group execution failed: {result_group}")
                    continue
                all_results.extend(result_group)

        return all_results

    def _get_tasks_by_ids(self, task_ids: list[str], prep_res: dict) -> list[Task]:
        """Hole Task-Objekte basierend auf IDs"""
        all_tasks = prep_res["all_tasks"]
        return [all_tasks[tid] for tid in task_ids if tid in all_tasks]

    def _is_critical_task(self, task_id: str, prep_res: dict) -> bool:
        """Prüfe ob Task kritisch ist"""
        task = prep_res["all_tasks"].get(task_id)
        if not task:
            return False
        return getattr(task, 'critical', False) or task.priority == 1

    async def _execute_parallel_batch(self, tasks: list[Task]) -> list[dict]:
        """Führe Tasks parallel aus"""
        if not tasks:
            return []

        # Limitiere auf max_parallel
        batch_size = min(len(tasks), self.max_parallel)
        batches = [tasks[i:i + batch_size] for i in range(0, len(tasks), batch_size)]

        all_results = []
        for batch in batches:
            batch_results = await asyncio.gather(
                *[self._execute_single_task(task) for task in batch],
                return_exceptions=True
            )

            # Handle exceptions
            processed_results = []
            for i, result in enumerate(batch_results):
                if isinstance(result, Exception):
                    eprint(f"Task {batch[i].id} failed with exception: {result}")
                    processed_results.append({
                        "task_id": batch[i].id,
                        "status": "failed",
                        "error": str(result)
                    })
                else:
                    processed_results.append(result)

            all_results.extend(processed_results)

        return all_results

    async def _execute_sequential_batch(self, tasks: list[Task]) -> list[dict]:
        """Führe Tasks sequenziell aus"""
        results = []

        for task in tasks:
            try:
                result = await self._execute_single_task(task)
                results.append(result)

                # Stoppe bei kritischen Fehlern in sequenzieller Ausführung
                if result.get("status") == "failed" and getattr(task, 'critical', False):
                    eprint(f"Critical task {task.id} failed, stopping sequential execution")
                    break

            except Exception as e:
                eprint(f"Sequential task {task.id} failed: {e}")
                results.append({
                    "task_id": task.id,
                    "status": "failed",
                    "error": str(e)
                })

                if getattr(task, 'critical', False):
                    break

        return results

    async def _execute_single_task(self, task: Task) -> dict:
        """Enhanced task execution with unified LLMToolNode usage"""
        if self.progress_tracker:
            await self.progress_tracker.emit_event(ProgressEvent(
                event_type="task_start",
                node_name="TaskExecutorNode",
                status=NodeStatus.RUNNING,
                task_id=task.id,
                plan_id=self.variable_manager.get("shared.current_plan.id"),
                metadata={
                    "description": task.description,
                    "type": task.type,
                    "priority": task.priority,
                    "dependencies": task.dependencies
                }
            ))

        task_start = time.perf_counter()
        try:
            task.status = "running"
            task.started_at = datetime.now()

            # Ensure metadata is initialized
            if not hasattr(task, 'metadata') or task.metadata is None:
                task.metadata = {}

            # Pre-process task with variable resolution
            if isinstance(task, ToolTask):
                resolved_args = self._resolve_task_variables(task.arguments)
                result = await self._execute_tool_task_with_validation(task, resolved_args)
            elif isinstance(task, LLMTask):
                # Use LLMToolNode for LLM tasks instead of direct execution
                result = await self._execute_llm_via_llmtool(task)
            elif isinstance(task, DecisionTask):
                # Enhanced decision task with context awareness
                result = await self._execute_decision_task_enhanced(task)
            else:
                # Use LLMToolNode for generic tasks as well
                result = await self._execute_generic_via_llmtool(task)

            # Store result in unified system
            self._store_task_result(task.id, result, True)

            task.result = result
            task.status = "completed"
            task.completed_at = datetime.now()

            task_duration = time.perf_counter() - task_start

            if self.progress_tracker:
                await self.progress_tracker.emit_event(ProgressEvent(
                    event_type="task_complete",
                    node_name="TaskExecutorNode",
                    task_id=task.id,
                    plan_id=self.variable_manager.get("shared.current_plan.id"),
                    status=NodeStatus.COMPLETED,
                    success=True,
                    duration=task_duration,
                    metadata={
                        "result_type": type(result).__name__,
                        "description": task.description
                    }
                ))

            return {
                "task_id": task.id,
                "status": "completed",
                "result": result
            }

        except Exception as e:
            task.error = str(e)
            task.status = "failed"
            task.retry_count += 1

            # Store error in unified system
            self._store_task_result(task.id, None, False, str(e))
            task_duration = time.perf_counter() - task_start

            if self.progress_tracker:
                await self.progress_tracker.emit_event(ProgressEvent(
                    event_type="task_error",  # Klarer Event-Typ
                    node_name="TaskExecutorNode",
                    task_id=task.id,
                    plan_id=self.variable_manager.get("shared.current_plan.id"),
                    status=NodeStatus.FAILED,
                    success=False,
                    duration=task_duration,
                    error_details={
                        "message": str(e),
                        "type": type(e).__name__
                    },
                    metadata={
                        "retry_count": task.retry_count,
                        "description": task.description
                    }
                ))

            eprint(f"Task {task.id} failed: {e}")
            return {
                "task_id": task.id,
                "status": "failed",
                "error": str(e),
                "retry_count": task.retry_count
            }

    async def _resolve_dynamic_arguments(self, arguments: dict[str, Any]) -> dict[str, Any]:
        """Enhanced dynamic argument resolution with full variable system"""
        resolved = {}

        for key, value in arguments.items():
            if isinstance(value, str):
                # FIXED: Use unified variable manager for all resolution
                resolved_value = self.variable_manager.format_text(value)

                # Log if variables weren't resolved (debugging)
                if "{{" in resolved_value and "}}" in resolved_value:
                    wprint(f"Unresolved variables in argument '{key}': {resolved_value}")

                resolved[key] = resolved_value
            else:
                resolved[key] = value

        return resolved

    async def _execute_tool_task_with_validation(self, task: ToolTask, resolved_args: dict[str, Any]) -> Any:
        """Tool execution with improved error detection and validation"""

        if not task.tool_name:
            raise ValueError(f"ToolTask {task.id} missing tool_name")

        agent = self.agent_instance
        if not agent:
            raise ValueError("Agent instance not available for tool execution")

        tool_start = time.perf_counter()

        # Track tool call start
        if self.progress_tracker:
            await self.progress_tracker.emit_event(ProgressEvent(
                event_type="tool_call",
                timestamp=time.time(),
                node_name="TaskExecutorNode",
                status=NodeStatus.RUNNING,
                task_id=task.id,
                tool_name=task.tool_name,
                tool_args=resolved_args,
                metadata={
                    "task_type": "ToolTask",
                    "hypothesis": task.hypothesis,
                    "validation_criteria": task.validation_criteria
                }
            ))

        try:
            rprint(f"Executing tool {task.tool_name} with resolved args: {resolved_args}")

            # Execute tool with timeout and retry logic
            result = await self._execute_tool_with_retries(task.tool_name, resolved_args, agent)

            tool_duration = time.perf_counter() - tool_start

            # Validate result before marking as success
            is_valid_result = self._validate_tool_result(result, task)

            if not is_valid_result:
                raise ValueError(f"Tool {task.tool_name} returned invalid result: {type(result).__name__}")

            # Track successful tool call
            if self.progress_tracker:
                await self.progress_tracker.emit_event(ProgressEvent(
                    event_type="tool_call",
                    timestamp=time.time(),
                    node_name="TaskExecutorNode",
                    task_id=task.id,
                    status=NodeStatus.COMPLETED,
                    tool_name=task.tool_name,
                    tool_args=resolved_args,
                    tool_result=result,
                    duration=tool_duration,
                    success=True,
                    metadata={
                        "task_type": "ToolTask",
                        "result_type": type(result).__name__,
                        "result_length": len(str(result)),
                        "validation_passed": is_valid_result
                    }
                ))

            # FIXED: Store in variable manager with correct path structure
            if self.variable_manager:
                self.variable_manager.set(f"results.{task.id}.data", result)
                self.variable_manager.set(f"tasks.{task.id}.result", result)

            return result

        except Exception as e:
            tool_duration = time.perf_counter() - tool_start
            import traceback
            print(traceback.format_exc())

            # Detailed error tracking
            if self.progress_tracker:
                await self.progress_tracker.emit_event(ProgressEvent(
                    event_type="tool_call",
                    timestamp=time.time(),
                    node_name="TaskExecutorNode",
                    task_id=task.id,
                    status=NodeStatus.FAILED,
                    tool_name=task.tool_name,
                    tool_args=resolved_args,
                    duration=tool_duration,
                    success=False,
                    tool_error=str(e),
                    metadata={
                        "task_type": "ToolTask",
                        "error_type": type(e).__name__,
                        "retry_attempted": hasattr(self, '_retry_count')
                    }
                ))

            eprint(f"Tool execution failed for {task.tool_name}: {e}")
            raise
    async def _execute_llm_via_llmtool(self, task: LLMTask) -> Any:
        """Execute LLM task via LLMToolNode for consistency"""

        # Prepare context for LLMToolNode
        llm_shared = {
            "current_task_description": task.description,
            "formatted_context": {
                "recent_interaction": f"Executing LLM task: {task.description}",
                "session_summary": "",
                "task_context": f"Task ID: {task.id}, Priority: {task.priority}"
            },
            "variable_manager": self.variable_manager,
            "agent_instance": self.agent_instance,
            "available_tools": self.agent_instance.shared.get("available_tools", []) if self.agent_instance else [],
            "tool_capabilities": self.agent_instance._tool_capabilities if self.agent_instance else {},
            "fast_llm_model": self.fast_llm_model,
            "complex_llm_model": self.complex_llm_model,
            "progress_tracker": self.progress_tracker,
            "session_id": getattr(self, 'session_id', 'task_executor'),
            "use_fast_response": task.llm_config.get("model_preference", "fast") == "fast"
        }

        # Create LLMToolNode instance
        llm_node = LLMToolNode()

        # Execute via LLMToolNode
        try:
            result = await llm_node.run_async(llm_shared)
            # shared["current_response"]
            # shared["tool_calls_made"]
            # shared["llm_tool_conversation"]
            # shared["synthesized_response"]
            return llm_shared["current_response"]
        except Exception as e:
            eprint(f"LLMToolNode execution failed for task {task.id}: {e}")
            # Fallback to direct execution
            import traceback
            print(traceback.format_exc())
            return await self._execute_llm_task_enhanced(task)

    async def _execute_llm_task_enhanced(self, task: LLMTask) -> Any:
        """Enhanced LLM task execution with unified variable system"""
        if not LITELLM_AVAILABLE:
            raise Exception("LiteLLM not available for LLM tasks")

        # Get model preference with variable support
        llm_config = task.llm_config
        model_preference = llm_config.get("model_preference", "fast")

        if model_preference == "complex":
            model_to_use = self.variable_manager.get("system.complex_llm_model", "openrouter/openai/gpt-4o")
        else:
            model_to_use = self.variable_manager.get("system.fast_llm_model", "openrouter/anthropic/claude-3-haiku")

        # Build context for prompt
        context_data = {}
        for context_key in task.context_keys:
            value = self.variable_manager.get(context_key)
            if value is not None:
                context_data[context_key] = value

        # Resolve prompt template with full variable system
        final_prompt = self.variable_manager.format_text(
            task.prompt_template,
            context=context_data
        )

        llm_start = time.perf_counter()

        try:

            response = await litellm.acompletion(
                model=model_to_use,
                messages=[{"role": "user", "content": final_prompt}],
                temperature=llm_config.get("temperature", 0.7),
                max_tokens=llm_config.get("max_tokens", 2048)
            )

            result = response


            # Store intermediate result for other tasks
            self.variable_manager.set(f"tasks.{task.id}.result", result)

            # Output schema validation if present
            if task.output_schema:
                stripped = result.strip()

                try:
                    # Try JSON first if it looks like JSON
                    if stripped.startswith('{') or stripped.startswith('['):
                        parsed = json.loads(stripped)
                    else:
                        parsed = yaml.safe_load(stripped)

                    # Ensure metadata is a dict before updating
                    if not isinstance(task.metadata, dict):
                        task.metadata = {}

                    # Save parsed result
                    task.metadata["parsed_output"] = parsed

                except (json.JSONDecodeError, yaml.YAMLError):
                    # Save info about failure without logging output
                    if not isinstance(task.metadata, dict):
                        task.metadata = {}
                    task.metadata["parsed_output_error"] = "Invalid JSON/YAML format"

                except Exception as e:
                    if not isinstance(task.metadata, dict):
                        task.metadata = {}
                    task.metadata["parsed_output_error"] = f"Unexpected error: {str(e)}"

            return result
        except Exception as e:
            llm_duration = time.perf_counter() - llm_start

            if self.progress_tracker:
                await self.progress_tracker.emit_event(ProgressEvent(
                    event_type="llm_call",
                    node_name="TaskExecutorNode",
                    task_id=task.id,
                    status=NodeStatus.FAILED,
                    success=False,
                    duration=llm_duration,
                    llm_model=model_to_use,
                    error_details={
                        "message": str(e),
                        "type": type(e).__name__
                    }
                ))

            raise

    async def _execute_generic_via_llmtool(self, task: Task) -> Any:
        """
        Execute a generic task by treating its description as a query for the LLMToolNode.
        This provides a flexible fallback for undefined task types, leveraging the full
        reasoning and tool-use capabilities of the LLMToolNode.
        """
        # Prepare a shared context dictionary for the LLMToolNode, treating the
        # generic task's description as the primary query.
        llm_shared = {
            "current_task_description": task.description,
            "current_query": task.description,
            "formatted_context": {
                "recent_interaction": f"Executing generic task: {task.description}",
                "session_summary": f"The system needs to complete the following task: {task.description}",
                "task_context": f"Task ID: {task.id}, Priority: {task.priority}, Type: Generic"
            },
            "variable_manager": self.variable_manager,
            "agent_instance": self.agent_instance,
            # Generic tasks might require tools, so provide full tool context.
            "available_tools": self.agent_instance.shared.get("available_tools", []) if self.agent_instance else [],
            "tool_capabilities": self.agent_instance._tool_capabilities if self.agent_instance else {},
            "fast_llm_model": self.fast_llm_model,
            "complex_llm_model": self.complex_llm_model,
            "progress_tracker": self.progress_tracker,
            "session_id": getattr(self, 'session_id', 'task_executor_generic'),
            # Default to a fast model, assuming generic tasks are often straightforward.
            "use_fast_response": True
        }

        # Instantiate the LLMToolNode for this specific execution.
        llm_node = LLMToolNode()

        try:
            # Execute the node. It will run its internal loop for reasoning, tool calling, and response generation.
            # The results of the execution will be populated back into the `llm_shared` dictionary.
            await llm_node.run_async(llm_shared)

            # Extract the final response from the shared context populated by the node.
            # Prioritize the structured 'synthesized_response' but fall back to 'current_response'.
            final_response = llm_shared.get("synthesized_response", {}).get("synthesized_response")
            if not final_response:
                final_response = llm_shared.get("current_response", f"Generic task '{task.id}' processed.")

            return final_response

        except Exception as e:
            eprint(f"LLMToolNode execution for generic task {task.id} failed: {e}")
            # Re-raise the exception to allow the higher-level execution loop in
            # _execute_single_task to catch and handle it appropriately (e.g., for retries).
            raise

    async def _execute_decision_task_enhanced(self, task: DecisionTask) -> str:
        """Enhanced DecisionTask with intelligent replan assessment"""

        if not LITELLM_AVAILABLE:
            raise Exception("LiteLLM not available for decision tasks")

        # Build comprehensive context for decision
        decision_context = self._build_decision_context(task)

        # Enhanced decision prompt with full context
        enhanced_prompt = f"""
You are making a critical routing decision for task execution. Analyze all context carefully.

## Current Situation
{task.decision_prompt}

## Execution Context
{decision_context}

## Available Routing Options
{json.dumps(task.routing_map, indent=2)}

## Decision Guidelines
1. Only trigger "replan_from_here" if there's a genuine failure that cannot be recovered
2. Use "route_to_task" for normal flow continuation
3. Consider the full context, not just immediate results
4. Be conservative with replanning - it's expensive and can cause loops

Based on ALL the context above, what is your decision?
Respond with EXACTLY one of these options: {', '.join(task.routing_map.keys())}

Your decision:"""

        model_to_use = self.fast_llm_model if hasattr(self, 'fast_llm_model') else "openrouter/anthropic/claude-3-haiku"

        try:
            response = await litellm.acompletion(
                model=model_to_use,
                messages=[{"role": "user", "content": enhanced_prompt}],
                temperature=0.1,
                max_tokens=50
            )

            decision = response.choices[0].message.content.strip().lower().split('\n')[0]

            # Find matching key (case-insensitive)
            matched_key = None
            for key in task.routing_map:
                if key.lower() == decision:
                    matched_key = key
                    break

            if not matched_key:
                wprint(f"Decision '{decision}' not in routing map, using first option")
                matched_key = list(task.routing_map.keys())[0] if task.routing_map else "continue"

            routing_instruction = task.routing_map.get(matched_key, matched_key)

            # Enhanced metadata with decision reasoning
            if not hasattr(task, 'metadata'):
                task.metadata = {}

            task.metadata.update({
                "decision_made": matched_key,
                "routing_instruction": routing_instruction,
                "decision_context": decision_context,
                "replan_justified": self._assess_replan_necessity(matched_key, routing_instruction, decision_context)
            })

            # Handle dynamic planning instructions
            if isinstance(routing_instruction, dict) and "action" in routing_instruction:
                action = routing_instruction["action"]

                if action == "replan_from_here":
                    # Add extensive context for replanning
                    task.metadata["replan_context"] = {
                        "new_goal": routing_instruction.get("new_goal", "Continue with alternative approach"),
                        "failure_reason": f"Decision task {task.id} determined: {matched_key}",
                        "original_task": task.id,
                        "context": routing_instruction.get("context", ""),
                        "execution_history": self._get_execution_history_summary(),
                        "failed_approaches": self._identify_failed_approaches(),
                        "success_indicators": self._identify_success_patterns()
                    }

                self.variable_manager.set(f"tasks.{task.id}.result", {
                    "decision": matched_key,
                    "action": action,
                    "instruction": routing_instruction,
                    "confidence": self._calculate_decision_confidence(decision_context)
                })

                return action

            else:
                # Traditional routing
                next_task_id = routing_instruction if isinstance(routing_instruction, str) else str(routing_instruction)

                task.metadata.update({
                    "next_task_id": next_task_id,
                    "routing_action": "route_to_task"
                })

                self.variable_manager.set(f"tasks.{task.id}.result", {
                    "decision": matched_key,
                    "next_task": next_task_id
                })

                return matched_key

        except Exception as e:
            eprint(f"Enhanced decision task failed: {e}")
            raise

    async def post_async(self, shared, prep_res, exec_res):
        """Erweiterte Post-Processing mit dynamischer Plan-Anpassung"""

        # Results store in shared state integrieren
        shared["results"] = self.results_store

        if exec_res is None or "error" in exec_res:
            shared["executor_performance"] = {"status": "error", "last_error": exec_res.get("error")}
            return "execution_error"

        if exec_res["status"] == "waiting":
            shared["executor_status"] = "waiting_for_dependencies"
            return "waiting"

        # Performance-Metriken speichern
        performance_data = {
            "execution_duration": exec_res.get("execution_duration", 0),
            "strategy_used": exec_res.get("strategy_used", "unknown"),
            "completed_tasks": exec_res.get("completed_tasks", 0),
            "failed_tasks": exec_res.get("failed_tasks", 0),
            "success_rate": exec_res.get("completed_tasks", 0) / max(len(exec_res.get("results", [])), 1),
            "timestamp": datetime.now().isoformat()
        }
        shared["executor_performance"] = performance_data

        # Check for dynamic planning actions
        planning_action_detected = False

        for result in exec_res.get("results", []):
            task_id = result["task_id"]
            if task_id in shared["tasks"]:
                task = shared["tasks"][task_id]
                task.status = result["status"]

                if result["status"] == "completed":
                    task.result = result["result"]

                    # Check for planning actions from DecisionTasks
                    if hasattr(task, 'metadata') and task.metadata:
                        routing_action = task.metadata.get("routing_action")

                        if routing_action == "replan_from_here":
                            shared["needs_dynamic_replan"] = True
                            shared["replan_context"] = task.metadata.get("replan_context", {})
                            planning_action_detected = True
                            rprint(f"Dynamic replan triggered by task {task_id}")

                        elif routing_action == "append_plan":
                            shared["needs_plan_append"] = True
                            shared["append_context"] = task.metadata.get("append_context", {})
                            planning_action_detected = True
                            rprint(f"Plan append triggered by task {task_id}")

                    # Store verification results if available
                    if result.get("verification"):
                        if not hasattr(task, 'metadata'):
                            task.metadata = {}
                        task.metadata["verification"] = result["verification"]

                elif result["status"] == "failed":
                    task.error = result.get("error", "Unknown error")

        # Return appropriate status based on planning actions
        if planning_action_detected:
            if shared.get("needs_dynamic_replan"):
                return "needs_dynamic_replan"  # Goes to PlanReflectorNode
            elif shared.get("needs_plan_append"):
                return "needs_plan_append"  # Goes to PlanReflectorNode

        # Regular completion checking
        current_plan = shared["current_plan"]
        if current_plan:
            all_finished = all(
                shared["tasks"][task.id].status in ["completed", "failed"]
                for task in current_plan.tasks
            )

            if all_finished:
                current_plan.status = "completed"
                shared["plan_completion_time"] = datetime.now().isoformat()
                rprint(f"Plan {current_plan.id} finished")
                return "plan_completed"
            else:
                ready_tasks = [
                    task for task in current_plan.tasks
                    if shared["tasks"][task.id].status == "pending"
                ]

                if ready_tasks:
                    return "continue_execution"
                else:
                    return "waiting"

        return "execution_complete"

    def get_execution_statistics(self) -> dict[str, Any]:
        """Erhalte detaillierte Ausführungsstatistiken"""
        if not self.execution_history:
            return {"message": "No execution history available"}

        history = self.execution_history

        return {
            "total_executions": len(history),
            "average_duration": sum(h["duration"] for h in history) / len(history),
            "success_rate": sum(1 for h in history if h["success"]) / len(history),
            "strategy_usage": {
                strategy: sum(1 for h in history if h["strategy"] == strategy)
                for strategy in set(h["strategy"] for h in history)
            },
            "total_tasks_executed": sum(h["tasks_executed"] for h in history),
            "average_confidence": sum(h["plan_confidence"] for h in history) / len(history),
            "recent_performance": history[-3:] if len(history) >= 3 else history
        }

    def _resolve_task_variables(self, data):
        """Unified variable resolution for any task data"""
        if isinstance(data, str):
            res = self.variable_manager.format_text(data)
            return res
        elif isinstance(data, dict):
            resolved = {}
            for key, value in data.items():
                resolved[key] = self._resolve_task_variables(value)
            return resolved
        elif isinstance(data, list):
            return [self._resolve_task_variables(item) for item in data]
        else:
            return data

    def _store_task_result(self, task_id: str, result: Any, success: bool, error: str = None):
        """Store task result in unified variable system"""
        result_data = {
            "data": result,
            "metadata": {
                "task_type": "task",
                "completed_at": datetime.now().isoformat(),
                "success": success
            }
        }

        if error:
            result_data["error"] = error
            result_data["metadata"]["success"] = False

        # Store in results_store and update variable manager
        self.results_store[task_id] = result_data
        self.variable_manager.set_results_store(self.results_store)

        # FIXED: Store actual result data, not the wrapper object
        self.variable_manager.set(f"results.{task_id}.data", result)
        self.variable_manager.set(f"results.{task_id}.metadata", result_data["metadata"])
        if error:
            self.variable_manager.set(f"results.{task_id}.error", error)

    def _build_decision_context(self, task: DecisionTask) -> str:
        """Build comprehensive context for decision making"""

        context_parts = []

        # Recent execution results
        recent_results = []
        for task_id, result_data in list(self.results_store.items())[-3:]:
            success = result_data.get("metadata", {}).get("success", False)
            status = "✓" if success else "✗"
            data_preview = str(result_data.get("data", ""))[:100] + "..."
            recent_results.append(f"{status} {task_id}: {data_preview}")

        if recent_results:
            context_parts.append("Recent Results:\n" + "\n".join(recent_results))

        # Variable context
        if self.variable_manager:
            available_vars = list(self.variable_manager.get_available_variables().keys())[:10]
            context_parts.append(f"Available Variables: {', '.join(available_vars)}")

        # Execution history
        execution_summary = self._get_execution_history_summary()
        if execution_summary:
            context_parts.append(f"Execution Summary: {execution_summary}")

        # Current world model insights
        world_insights = self._get_world_model_insights()
        if world_insights:
            context_parts.append(f"Known Facts: {world_insights}")

        return "\n\n".join(context_parts)

    def _assess_replan_necessity(self, decision: str, routing_instruction: Any, context: str) -> bool:
        """Assess if replanning is truly necessary"""

        if not isinstance(routing_instruction, dict):
            return False

        action = routing_instruction.get("action", "")
        if action != "replan_from_here":
            return False

        # Check if we have genuine failures
        genuine_failures = "error" in context.lower() or "failed" in context.lower()
        alternative_available = len(self.results_store) > 0  # Have some results to work with

        # Be conservative - only replan if really necessary
        return genuine_failures and not alternative_available

    async def _execute_tool_with_retries(self, tool_name: str, args: dict, agent, max_retries: int = 2) -> Any:
        """Execute tool with retry logic"""

        last_exception = None

        for attempt in range(max_retries + 1):
            try:
                result = await agent.arun_function(tool_name, **args)

                # Additional validation - check if result indicates success
                if self._is_tool_result_success(result):
                    return result
                elif attempt < max_retries:
                    wprint(f"Tool {tool_name} returned unclear result, retrying...")
                    continue
                else:
                    return result

            except Exception as e:
                last_exception = e
                if attempt < max_retries:
                    wprint(f"Tool {tool_name} failed (attempt {attempt + 1}), retrying: {e}")
                    # await asyncio.sleep(0.5 * (attempt + 1))  # Progressive delay
                else:
                    eprint(f"Tool {tool_name} failed after {max_retries + 1} attempts")

        if last_exception:
            raise last_exception
        else:
            raise RuntimeError(f"Tool {tool_name} failed without exception")

    def _validate_tool_result(self, result: Any, task: ToolTask) -> bool:
        """Validate tool result to prevent false failures"""

        # Basic validation
        if result is None:
            return False

        # Check for common error indicators
        if isinstance(result, str):
            error_indicators = ["error", "failed", "exception", "timeout", "not found"]
            result_lower = result.lower()

            # If result contains error indicators but also has substantial content, it might still be valid
            has_errors = any(indicator in result_lower for indicator in error_indicators)
            has_content = len(result.strip()) > 20

            if has_errors and not has_content:
                return False

        # Check against expectation if provided
        if hasattr(task, 'expectation') and task.expectation:
            expectation_keywords = task.expectation.lower().split()
            result_text = str(result).lower()

            # At least one expectation keyword should be present
            if not any(keyword in result_text for keyword in expectation_keywords):
                wprint(f"Tool result doesn't match expectation: {task.expectation}")

        return True

    def _is_tool_result_success(self, result: Any) -> bool:
        """Determine if a tool result indicates success"""

        if result is None:
            return False

        if isinstance(result, bool):
            return result

        if isinstance(result, list | dict):
            return len(result) > 0

        if isinstance(result, str):
            # Check for explicit success/failure indicators
            result_lower = result.lower()

            success_indicators = ["success", "completed", "found", "retrieved", "generated"]
            failure_indicators = ["error", "failed", "not found", "timeout", "exception"]

            has_success = any(indicator in result_lower for indicator in success_indicators)
            has_failure = any(indicator in result_lower for indicator in failure_indicators)

            if has_success and not has_failure:
                return True
            elif has_failure and not has_success:
                return False
            else:
                # Ambiguous - assume success if there's substantial content
                return len(result.strip()) > 10

        # For other types, assume success if not None
        return True

    def _get_execution_history_summary(self) -> str:
        """Get concise execution history summary"""

        if not hasattr(self, 'execution_history') or not self.execution_history:
            return "No execution history"

        recent = self.execution_history[-3:]  # Last 3 executions
        summaries = []

        for hist in recent:
            status = "Success" if hist.get("success", False) else "Failed"
            duration = hist.get("duration", 0)
            strategy = hist.get("strategy", "Unknown")
            summaries.append(f"{strategy}: {status} ({duration:.1f}s)")

        return "; ".join(summaries)

    def _identify_failed_approaches(self) -> list[str]:
        """Identify approaches that have consistently failed"""

        failed_approaches = []

        # Analyze failed tasks
        for _task_id, result_data in self.results_store.items():
            if not result_data.get("metadata", {}).get("success", True):
                error = result_data.get("error", "")
                if "tool" in error.lower():
                    failed_approaches.append("direct_tool_approach")
                elif "search" in error.lower():
                    failed_approaches.append("search_based_approach")
                elif "llm" in error.lower():
                    failed_approaches.append("llm_direct_approach")

        return list(set(failed_approaches))

    def _identify_success_patterns(self) -> list[str]:
        """Identify patterns that have led to success"""

        success_patterns = []

        # Analyze successful tasks
        successful_results = [
            r for r in self.results_store.values()
            if r.get("metadata", {}).get("success", False)
        ]

        if successful_results:
            # Identify common patterns
            if len(successful_results) > 1:
                success_patterns.append("multi_step_approach")

            for result in successful_results:
                data = result.get("data", "")
                if isinstance(data, str) and len(data) > 100:
                    success_patterns.append("detailed_information_retrieval")

        return list(set(success_patterns))

    def _get_world_model_insights(self) -> str:
        """Get relevant insights from world model"""

        if not self.variable_manager:
            return ""

        world_data = self.variable_manager.scopes.get("world", {})
        if not world_data:
            return "No world model data"

        # Get most recent or relevant facts
        recent_facts = []
        for key, value in list(world_data.items())[:5]:  # Top 5 facts
            recent_facts.append(f"{key}: {str(value)[:50]}...")

        return "; ".join(recent_facts)

    def _calculate_decision_confidence(self, context: str) -> float:
        """Calculate confidence in decision based on context"""

        # Simple heuristic based on context richness
        base_confidence = 0.5

        # Boost confidence if we have rich context
        if len(context) > 200:
            base_confidence += 0.2

        # Boost if we have recent results
        if "Recent Results:" in context:
            base_confidence += 0.2

        # Reduce if there are many failures
        failure_count = context.lower().count("failed") + context.lower().count("error")
        base_confidence -= min(failure_count * 0.1, 0.3)

        return max(0.1, min(1.0, base_confidence))

exec_async(prep_res) async

Hauptausführungslogik mit intelligentem Routing

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

async def exec_async(self, prep_res):
    """Hauptausführungslogik mit intelligentem Routing"""

    if "error" in prep_res:
        return {"error": prep_res["error"]}

    execution_plan = prep_res["execution_plan"]

    if execution_plan["strategy"] == "waiting":
        return {
            "status": "waiting",
            "message": execution_plan["reason"],
            "blocked_count": execution_plan.get("blocked_count", 0)
        }

    # Starte Ausführung basierend auf Plan
    execution_start = datetime.now()

    try:
        if execution_plan["strategy"] == "parallel":
            results = await self._execute_parallel_plan(execution_plan, prep_res)
        elif execution_plan["strategy"] == "sequential":
            results = await self._execute_sequential_plan(execution_plan, prep_res)
        else:  # hybrid
            results = await self._execute_hybrid_plan(execution_plan, prep_res)

        execution_duration = (datetime.now() - execution_start).total_seconds()

        # Speichere Execution-History für LLM-Optimierung
        self.execution_history.append({
            "timestamp": execution_start.isoformat(),
            "strategy": execution_plan["strategy"],
            "duration": execution_duration,
            "tasks_executed": len(results),
            "success": all(r.get("status") == "completed" for r in results),
            "plan_confidence": execution_plan.get("confidence", 0.5)
        })

        # Behalte nur letzte 10 Executions
        if len(self.execution_history) > 10:
            self.execution_history = self.execution_history[-10:]

        return {
            "status": "executed",
            "results": results,
            "execution_duration": execution_duration,
            "strategy_used": execution_plan["strategy"],
            "completed_tasks": len([r for r in results if r.get("status") == "completed"]),
            "failed_tasks": len([r for r in results if r.get("status") == "failed"])
        }

    except Exception as e:
        eprint(f"Execution plan failed: {e}")
        return {
            "status": "execution_failed",
            "error": str(e),
            "results": []
        }

get_execution_statistics()

Erhalte detaillierte Ausführungsstatistiken

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

def get_execution_statistics(self) -> dict[str, Any]:
    """Erhalte detaillierte Ausführungsstatistiken"""
    if not self.execution_history:
        return {"message": "No execution history available"}

    history = self.execution_history

    return {
        "total_executions": len(history),
        "average_duration": sum(h["duration"] for h in history) / len(history),
        "success_rate": sum(1 for h in history if h["success"]) / len(history),
        "strategy_usage": {
            strategy: sum(1 for h in history if h["strategy"] == strategy)
            for strategy in set(h["strategy"] for h in history)
        },
        "total_tasks_executed": sum(h["tasks_executed"] for h in history),
        "average_confidence": sum(h["plan_confidence"] for h in history) / len(history),
        "recent_performance": history[-3:] if len(history) >= 3 else history
    }

post_async(shared, prep_res, exec_res) async

Erweiterte Post-Processing mit dynamischer Plan-Anpassung

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

async def post_async(self, shared, prep_res, exec_res):
    """Erweiterte Post-Processing mit dynamischer Plan-Anpassung"""

    # Results store in shared state integrieren
    shared["results"] = self.results_store

    if exec_res is None or "error" in exec_res:
        shared["executor_performance"] = {"status": "error", "last_error": exec_res.get("error")}
        return "execution_error"

    if exec_res["status"] == "waiting":
        shared["executor_status"] = "waiting_for_dependencies"
        return "waiting"

    # Performance-Metriken speichern
    performance_data = {
        "execution_duration": exec_res.get("execution_duration", 0),
        "strategy_used": exec_res.get("strategy_used", "unknown"),
        "completed_tasks": exec_res.get("completed_tasks", 0),
        "failed_tasks": exec_res.get("failed_tasks", 0),
        "success_rate": exec_res.get("completed_tasks", 0) / max(len(exec_res.get("results", [])), 1),
        "timestamp": datetime.now().isoformat()
    }
    shared["executor_performance"] = performance_data

    # Check for dynamic planning actions
    planning_action_detected = False

    for result in exec_res.get("results", []):
        task_id = result["task_id"]
        if task_id in shared["tasks"]:
            task = shared["tasks"][task_id]
            task.status = result["status"]

            if result["status"] == "completed":
                task.result = result["result"]

                # Check for planning actions from DecisionTasks
                if hasattr(task, 'metadata') and task.metadata:
                    routing_action = task.metadata.get("routing_action")

                    if routing_action == "replan_from_here":
                        shared["needs_dynamic_replan"] = True
                        shared["replan_context"] = task.metadata.get("replan_context", {})
                        planning_action_detected = True
                        rprint(f"Dynamic replan triggered by task {task_id}")

                    elif routing_action == "append_plan":
                        shared["needs_plan_append"] = True
                        shared["append_context"] = task.metadata.get("append_context", {})
                        planning_action_detected = True
                        rprint(f"Plan append triggered by task {task_id}")

                # Store verification results if available
                if result.get("verification"):
                    if not hasattr(task, 'metadata'):
                        task.metadata = {}
                    task.metadata["verification"] = result["verification"]

            elif result["status"] == "failed":
                task.error = result.get("error", "Unknown error")

    # Return appropriate status based on planning actions
    if planning_action_detected:
        if shared.get("needs_dynamic_replan"):
            return "needs_dynamic_replan"  # Goes to PlanReflectorNode
        elif shared.get("needs_plan_append"):
            return "needs_plan_append"  # Goes to PlanReflectorNode

    # Regular completion checking
    current_plan = shared["current_plan"]
    if current_plan:
        all_finished = all(
            shared["tasks"][task.id].status in ["completed", "failed"]
            for task in current_plan.tasks
        )

        if all_finished:
            current_plan.status = "completed"
            shared["plan_completion_time"] = datetime.now().isoformat()
            rprint(f"Plan {current_plan.id} finished")
            return "plan_completed"
        else:
            ready_tasks = [
                task for task in current_plan.tasks
                if shared["tasks"][task.id].status == "pending"
            ]

            if ready_tasks:
                return "continue_execution"
            else:
                return "waiting"

    return "execution_complete"

prep_async(shared) async

Enhanced preparation with unified variable system

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

async def prep_async(self, shared):
    """Enhanced preparation with unified variable system"""
    current_plan = shared.get("current_plan")
    tasks = shared.get("tasks", {})

    # Get unified variable manager
    self.variable_manager = shared.get("variable_manager")
    self.progress_tracker = shared.get("progress_tracker")
    if not self.variable_manager:
        self.variable_manager = VariableManager(shared.get("world_model", {}), shared)

    # Register all necessary scopes
    self.variable_manager.set_results_store(self.results_store)
    self.variable_manager.set_tasks_store(tasks)
    self.variable_manager.register_scope('user', shared.get('user_context', {}))
    self.variable_manager.register_scope('system', {
        'timestamp': datetime.now().isoformat(),
        'agent_name': shared.get('agent_instance', {}).amd.name if shared.get('agent_instance') else 'unknown'
    })

    # Stelle sicher, dass Agent-Referenz verfügbar ist
    if not self.agent_instance:
        self.agent_instance = shared.get("agent_instance")

    if not current_plan:
        return {"error": "No active plan", "tasks": tasks}

    # Rest of existing prep_async logic...
    ready_tasks = self._find_ready_tasks(current_plan, tasks)
    blocked_tasks = self._find_blocked_tasks(current_plan, tasks)

    execution_plan = await self._create_intelligent_execution_plan(
        ready_tasks, blocked_tasks, current_plan, shared
    )
    self.complex_llm_model = shared.get("complex_llm_model")
    self.fast_llm_model = shared.get("fast_llm_model")

    return {
        "plan": current_plan,
        "ready_tasks": ready_tasks,
        "blocked_tasks": blocked_tasks,
        "all_tasks": tasks,
        "execution_plan": execution_plan,
        "fast_llm_model": self.fast_llm_model,
        "complex_llm_model": self.complex_llm_model,
        "available_tools": shared.get("available_tools", []),
        "world_model": shared.get("world_model", {}),
        "results": self.results_store,
        "variable_manager": self.variable_manager,
        "progress_tracker": self.progress_tracker ,
    }

TaskManagementFlow

Bases: AsyncFlow

Enhanced Task-Management-Flow with LLMReasonerNode as strategic core. The flow now starts with strategic reasoning and delegates to specialized sub-systems.

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

@with_progress_tracking
class TaskManagementFlow(AsyncFlow):
    """
    Enhanced Task-Management-Flow with LLMReasonerNode as strategic core.
    The flow now starts with strategic reasoning and delegates to specialized sub-systems.
    """

    def __init__(self, max_parallel_tasks: int = 3, max_reasoning_loops: int = 24, max_tool_calls:int = 5):
        # Create the strategic reasoning core (new primary node)
        self.llm_reasoner = LLMReasonerNode(max_reasoning_loops=max_reasoning_loops)

        # Create specialized sub-system nodes (now supporting nodes)
        self.planner_node = TaskPlannerNode()
        self.executor_node = TaskExecutorNode(max_parallel=max_parallel_tasks)
        self.sync_node = StateSyncNode()
        self.llm_tool_node = LLMToolNode(max_tool_calls=max_tool_calls)

        # Store references for the reasoner to access sub-systems
        # These will be injected into shared state during execution

        # === NEW HIERARCHICAL FLOW STRUCTURE ===

        # Primary flow: LLMReasonerNode is the main orchestrator
        # It makes strategic decisions and routes to appropriate sub-systems

        # The reasoner can internally call any of these sub-systems:
        # - LLMToolNode for direct tool usage
        # - TaskPlanner + TaskExecutor for complex project management
        # - Direct response for simple queries

        # Only one main connection: reasoner completes -> response generation
        self.llm_reasoner - "reasoner_complete" >> self.sync_node

        # Fallback connections for error handling
        self.llm_reasoner - "error" >> self.sync_node
        self.llm_reasoner - "timeout" >> self.sync_node

        # The old linear connections are removed - the reasoner now controls the flow internally

        super().__init__(start=self.llm_reasoner)

    async def run_async(self, shared):
        """Enhanced run with sub-system injection"""

        # Inject sub-system references into shared state so reasoner can access them
        shared["llm_tool_node_instance"] = self.llm_tool_node
        shared["task_planner_instance"] = self.planner_node
        shared["task_executor_instance"] = self.executor_node

        # Store tool registry access for the reasoner
        agent_instance = shared.get("agent_instance")
        if agent_instance:
            shared["tool_registry"] = agent_instance._tool_registry
            shared["tool_capabilities"] = agent_instance._tool_capabilities

        # Execute the flow with the reasoner as starting point
        return await super().run_async(shared)

run_async(shared) async

Enhanced run with sub-system injection

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

async def run_async(self, shared):
    """Enhanced run with sub-system injection"""

    # Inject sub-system references into shared state so reasoner can access them
    shared["llm_tool_node_instance"] = self.llm_tool_node
    shared["task_planner_instance"] = self.planner_node
    shared["task_executor_instance"] = self.executor_node

    # Store tool registry access for the reasoner
    agent_instance = shared.get("agent_instance")
    if agent_instance:
        shared["tool_registry"] = agent_instance._tool_registry
        shared["tool_capabilities"] = agent_instance._tool_capabilities

    # Execute the flow with the reasoner as starting point
    return await super().run_async(shared)

TaskPlannerNode

Bases: AsyncNode

Erweiterte Aufgabenplanung mit dynamischen Referenzen und Tool-Integration

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

@with_progress_tracking
class TaskPlannerNode(AsyncNode):
    """Erweiterte Aufgabenplanung mit dynamischen Referenzen und Tool-Integration"""

    async def prep_async(self, shared):
        """Enhanced preparation with goals-based planning support"""

        # Check if this is a goals-based call from LLMReasonerNode
        replan_context = shared.get("replan_context", {})
        goals_list = replan_context.get("goals", [])

        if goals_list:
            # Goals-based planning (called by LLMReasonerNode)
            return {
                "goals": goals_list,
                "planning_mode": "goals_based",
                "query": shared.get("current_query", ""),
                "reasoning_context": replan_context.get("reasoning_context", ""),
                "triggered_by": replan_context.get("triggered_by", "unknown"),
                "tasks": shared.get("tasks", {}),
                "system_status": shared.get("system_status", "idle"),
                "tool_capabilities": shared.get("tool_capabilities", {}),
                "available_tools_names": shared.get("available_tools", []),
                "strategy": "goals_decomposition",  # New strategy type
                "fast_llm_model": shared.get("fast_llm_model"),
                "complex_llm_model": shared.get("complex_llm_model"),
                "agent_instance": shared.get("agent_instance"),
                "variable_manager": shared.get("variable_manager"),
            }
        else:
            # Legacy planning (original query-based approach)
            return {
                "query": shared.get("current_query", ""),
                "tasks": shared.get("tasks", {}),
                "system_status": shared.get("system_status", "idle"),
                "tool_capabilities": shared.get("tool_capabilities", {}),
                "available_tools_names": shared.get("available_tools", []),
                "strategy": shared.get("selected_strategy", "direct_response"),
                "fast_llm_model": shared.get("fast_llm_model"),
                "complex_llm_model": shared.get("complex_llm_model"),
                "agent_instance": shared.get("agent_instance"),
                "variable_manager": shared.get("variable_manager"),
                "planning_mode": "legacy"
            }

    async def exec_async(self, prep_res):
        if prep_res["strategy"] == "fast_simple_planning":
            return await self._create_simple_plan(prep_res)
        else:
            return await self._advanced_llm_decomposition(prep_res)

    async def post_async(self, shared, prep_res, exec_res):
        """Post-processing nach Plan-Erstellung"""

        if exec_res is None:
            shared["planning_error"] = "Plan creation returned None"
            return "planning_failed"

        if isinstance(exec_res, TaskPlan):

            progress_tracker = shared.get("progress_tracker")
            if progress_tracker:
                await progress_tracker.emit_event(ProgressEvent(
                    event_type="plan_created",
                    node_name="TaskPlannerNode",
                    session_id=shared.get("session_id"),
                    status=NodeStatus.COMPLETED,
                    success=True,
                    plan_id=exec_res.id,
                    metadata={
                        "plan_name": exec_res.name,
                        "task_count": len(exec_res.tasks),
                        "strategy": exec_res.execution_strategy
                    }
                ))

            # Erfolgreicher Plan
            shared["current_plan"] = exec_res

            # Tasks in shared state für Executor verfügbar machen
            task_dict = {task.id: task for task in exec_res.tasks}
            if "tasks" not in shared:
                shared["tasks"] = task_dict
            else:
                shared["tasks"].update(task_dict)

            # Plan-Metadaten setzen
            shared["plan_created_at"] = datetime.now().isoformat()
            shared["plan_strategy"] = exec_res.execution_strategy
            shared["total_tasks_planned"] = len(exec_res.tasks)

            rprint(f"Plan created successfully: {exec_res.name} with {len(exec_res.tasks)} tasks")
            return "planned"

        else:
            # Plan creation failed
            shared["planning_error"] = "Invalid plan format returned"
            shared["current_plan"] = None
            eprint("Plan creation failed - invalid format")
            return "planning_failed"

    async def _create_simple_plan(self, prep_res) -> TaskPlan:
        """Fast lightweight planning for direct or simple multi-step queries."""
        taw = self._build_tool_intelligence(prep_res)
        rprint("You are a FAST "+ taw)
        prompt = f"""
You are a FAST abstract pattern recognizer and task planner.
Identify if the query needs a **single-step LLM answer** or a **simple 2–3 task plan** using available tools.
Output ONLY YAML.

## User Query
{prep_res['query']}

## Available Tools
{taw}

## Pattern Recognition (Internal Only)
- Detect if query is informational, action-based, or tool-eligible.
- Map to minimal plan type: "direct_llm" or "simple_tool_plus_llm".

## YAML Schema
```yaml
plan_name: string
description: string
execution_strategy: "sequential" | "parallel"
tasks:
  - id: string
    type: "LLMTask" | "ToolTask"
    description: string
    priority: int
    dependencies: [list]
Example 1 — Direct LLM
```yaml
plan_name: Direct Response
description: Quick answer from LLM
execution_strategy: sequential
tasks:
  - id: answer
    type: LLMTask
    description: Respond to query
    priority: 1
    dependencies: []
    prompt_template: Respond concisely to: {prep_res['query']}
    llm_config:
      model_preference: fast
      temperature: 0.3
```
Example 2 — Tool + LLM
```yaml
plan_name: Fetch and Answer
description: Get info from tool and summarize
execution_strategy: sequential
tasks:
  - id: fetch_info
    type: ToolTask
    description: Get required data
    priority: 1
    dependencies: []
    tool_name: info_api
    arguments:
      query: "{prep_res['query']}"
  - id: summarize
    type: LLMTask
    description: Summarize fetched data
    priority: 2
    dependencies: ["fetch_info"]
    prompt_template: Summarize: {{ results.fetch_info.data }}
    llm_config:
      model_preference: fast
      temperature: 0.3
```
Output Requirements
Use ONLY YAML for the final output
Pick minimal plan type for fastest completion!
focus on correct quotation and correct yaml format!
    """

        try:
            agent_instance = prep_res["agent_instance"]
            plan_data = await agent_instance.a_format_class(PlanData,
                model=prep_res.get("complex_llm_model", "openrouter/anthropic/claude-3-haiku"),
                prompt= prompt,
                temperature=0.3,
                max_tokens=4512,
                auto_context=True,
                node_name="TaskPlannerNode",
                task_id="fast_simple_planning"
            )
            # print("Simple", json.dumps(plan_data, indent=2))
            return TaskPlan(
                id=str(uuid.uuid4()),
                name=plan_data.get("plan_name", "Generated Plan"),
                description=plan_data.get("description", f"Plan for: {prep_res['query']}"),
                tasks=[
                    [LLMTask, ToolTask, DecisionTask, Task][["LLMTask", "ToolTask", "DecisionTask", "Task"].index(t.get("type"))](**t)
                    for t in plan_data.get("tasks", [])
                ],
                execution_strategy=plan_data.get("execution_strategy", "sequential")
            )

        except Exception as e:
            eprint(f"Simple plan creation failed: {e}")
            import traceback
            print(traceback.format_exc())
            return TaskPlan(
                id=str(uuid.uuid4()),
                name="Fallback Plan",
                description="Direct response only",
                tasks=[
                    LLMTask(
                        id="fast_simple_planning",
                        type="LLMTask",
                        description="Generate direct response",
                        priority=1,
                        dependencies=[],
                        prompt_template=f"Respond to the query: {prep_res['query']}",
                        llm_config={"model_preference": "fast"}
                    )
                ]
            )

    async def _advanced_llm_decomposition(self, prep_res) -> TaskPlan:
        """Enhanced LLM-based decomposition with goals-based planning support"""

        planning_mode = prep_res.get("planning_mode", "legacy")
        variable_manager = prep_res.get("variable_manager")
        tool_intelligence = self._build_tool_intelligence(prep_res)

        if planning_mode == "goals_based":
            # Goals-based planning from LLMReasonerNode
            goals_list = prep_res.get("goals", [])
            reasoning_context = prep_res.get("reasoning_context", "")

            prompt = f"""
You are an expert task planner specialized in creating execution plans from strategic goals.
Create a comprehensive plan that addresses all goals with proper dependencies and parallelization.

## Strategic Goals from Reasoner
{chr(10).join([f"{i + 1}. {goal}" for i, goal in enumerate(goals_list)])}

## Reasoning Context
{reasoning_context}

## Your Available Tools & Intelligence
{tool_intelligence}

{variable_manager.get_llm_variable_context() if variable_manager else ""}

## Goals-Based Planning Instructions
1. Analyze each goal for dependencies on other goals
2. Identify goals that can be executed in parallel
3. Create tasks that address each goal effectively
4. Use variable references {{ results.task_id.data }} for dependencies
5. Ensure proper sequencing and coordination

## YAML Schema
```yaml
plan_name: string
description: string
execution_strategy: "sequential" | "parallel" | "mixed"
tasks:
  - id: string
    type: "LLMTask" | "ToolTask" | "DecisionTask"
    description: string
    priority: int
    dependencies: [list of task ids]
    # Type-specific fields as needed
Goals Decomposition Strategy

Independent Goals: Create parallel tasks
Sequential Goals: Use dependencies array
Complex Goals: Break into sub-tasks with DecisionTask routing
Data Dependencies: Use variable references between tasks

Example for Multi-Goal Plan
yamlCopyplan_name: "Multi-Goal Strategic Plan"
description: "Execute multiple strategic objectives with proper coordination"
execution_strategy: "mixed"
tasks:
  - id: "goal_1_research"
    type: "ToolTask"
    description: "Research data for Goal 1"
    priority: 1
    dependencies: []
    tool_name: "search_web"
    arguments:
      query: "research topic for goal 1"

  - id: "goal_2_research"
    type: "ToolTask"
    description: "Research data for Goal 2"
    priority: 1
    dependencies: []
    tool_name: "search_web"
    arguments:
      query: "research topic for goal 2"

  - id: "analyze_combined"
    type: "LLMTask"
    description: "Analyze combined research results"
    priority: 2
    dependencies: ["goal_1_research", "goal_2_research"]
    prompt_template: |
      Analyze these research results:
      Goal 1 Data: {{ results.goal_1_research.data }}
      Goal 2 Data: {{ results.goal_2_research.data }}

      Provide comprehensive analysis addressing both goals.
    llm_config:
      model_preference: "complex"
      temperature: 0.3
Generate the execution plan for the strategic goals:
    """

        else:
            # Legacy single-query planning
            base_query = prep_res['query']
            prompt = f"""
You are an expert task planner with dynamic adaptation capabilities.
Create intelligent, adaptive execution plans for the user query.
User Query
{base_query}
Your Available Tools & Intelligence
{tool_intelligence}
{variable_manager.get_llm_variable_context() if variable_manager else ""}
TASK TYPES (Dataclass-Aligned)

LLMTask: Step that uses a language model
ToolTask: Step that calls an available tool
DecisionTask: Step that decides routing between tasks

YAML SCHEMA
yamlCopyplan_name: string
description: string
execution_strategy: "sequential" | "parallel" | "mixed"
tasks:
  - id: string
    type: "LLMTask" | "ToolTask" | "DecisionTask"
    description: string
    priority: int
    dependencies: [list of task ids]
    # Additional fields depending on type
Generate the adaptive execution plan:
            """

        try:
            model_to_use = prep_res.get("complex_llm_model", "openrouter/openai/gpt-4o")
            agent_instance = prep_res["agent_instance"]

            plan_data = await agent_instance.a_format_class(PlanData,
                model=model_to_use,
                prompt= prompt,
                temperature=0.3,
                auto_context=True,
                node_name="TaskPlannerNode",
                task_id="goals_based_planning" if planning_mode == "goals_based" else "adaptive_planning"
            )
            # Create specialized tasks
            tasks = []
            for task_data in plan_data.get("tasks", []):
                task_type = task_data.pop("type", "generic")
                task = create_task(task_type, **task_data)
                tasks.append(task)

            plan = TaskPlan(
                id=str(uuid.uuid4()),
                name=plan_data.get("plan_name", "Generated Plan"),
                description=plan_data.get("description",
                                          "Plan for goals-based execution" if planning_mode == "goals_based" else f"Plan for: {base_query}"),
                tasks=tasks,
                execution_strategy=plan_data.get("execution_strategy", "sequential"),
                metadata={
                    "planning_mode": planning_mode,
                    "goals_count": len(prep_res.get("goals", [])) if planning_mode == "goals_based" else 1
                }
            )

            rprint(f"Created {planning_mode} plan with {len(tasks)} tasks")
            return plan

        except Exception as e:
            eprint(f"Advanced planning failed: {e}")
            import traceback

            print(traceback.format_exc())
            return await self._create_simple_plan(prep_res)

    def _build_tool_intelligence(self, prep_res: dict) -> str:
        """Build detailed tool intelligence for planning"""

        agent_instance = prep_res.get("agent_instance")
        if not agent_instance or not hasattr(agent_instance, '_tool_capabilities'):
            return "No tool intelligence available."

        capabilities = agent_instance._tool_capabilities
        query = prep_res.get('query', '').lower()

        context_parts = []
        context_parts.append("### Intelligent Tool Analysis:")

        for tool_name, cap in capabilities.items():
            context_parts.append(f"\n{tool_name}:")
            context_parts.append(f"- Function: {cap.get('primary_function', 'Unknown')}")
            context_parts.append(f"- Arguments: {yaml.dump(cap.get('args_schema', 'takes no arguments!'), default_flow_style=False)}")

            # Check relevance to current query
            relevance_score = self._calculate_tool_relevance(query, cap)
            context_parts.append(f"- Query relevance: {relevance_score:.2f}")

            if relevance_score > 0.4:
                context_parts.append("- ⭐ HIGHLY RELEVANT - SHOULD USE THIS TOOL!")

            # Show trigger analysis
            triggers = cap.get('trigger_phrases', [])
            matched_triggers = [t for t in triggers if t.lower() in query]
            if matched_triggers:
                context_parts.append(f"- Matched triggers: {matched_triggers}")

            # Show use cases
            use_cases = cap.get('use_cases', [])[:3]
            context_parts.append(f"- Use cases: {', '.join(use_cases)}")

        return "\n".join(context_parts)

    def _calculate_tool_relevance(self, query: str, capabilities: dict) -> float:
        """Calculate how relevant a tool is to the current query"""

        query_words = set(query.lower().split())

        # Check trigger phrases
        trigger_score = 0.0
        triggers = capabilities.get('trigger_phrases', [])
        for trigger in triggers:
            trigger_words = set(trigger.lower().split())
            if trigger_words.intersection(query_words):
                trigger_score += 0.04
        # Check confidence triggers if available
        conf_triggers = capabilities.get('confidence_triggers', {})
        for phrase, confidence in conf_triggers.items():
            if phrase.lower() in query:
                trigger_score += confidence/10
        # Check indirect connections
        indirect = capabilities.get('indirect_connections', [])
        for connection in indirect:
            connection_words = set(connection.lower().split())
            if connection_words.intersection(query_words):
                trigger_score += 0.02
        return min(1.0, trigger_score)

post_async(shared, prep_res, exec_res) async

Post-processing nach Plan-Erstellung

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

async def post_async(self, shared, prep_res, exec_res):
    """Post-processing nach Plan-Erstellung"""

    if exec_res is None:
        shared["planning_error"] = "Plan creation returned None"
        return "planning_failed"

    if isinstance(exec_res, TaskPlan):

        progress_tracker = shared.get("progress_tracker")
        if progress_tracker:
            await progress_tracker.emit_event(ProgressEvent(
                event_type="plan_created",
                node_name="TaskPlannerNode",
                session_id=shared.get("session_id"),
                status=NodeStatus.COMPLETED,
                success=True,
                plan_id=exec_res.id,
                metadata={
                    "plan_name": exec_res.name,
                    "task_count": len(exec_res.tasks),
                    "strategy": exec_res.execution_strategy
                }
            ))

        # Erfolgreicher Plan
        shared["current_plan"] = exec_res

        # Tasks in shared state für Executor verfügbar machen
        task_dict = {task.id: task for task in exec_res.tasks}
        if "tasks" not in shared:
            shared["tasks"] = task_dict
        else:
            shared["tasks"].update(task_dict)

        # Plan-Metadaten setzen
        shared["plan_created_at"] = datetime.now().isoformat()
        shared["plan_strategy"] = exec_res.execution_strategy
        shared["total_tasks_planned"] = len(exec_res.tasks)

        rprint(f"Plan created successfully: {exec_res.name} with {len(exec_res.tasks)} tasks")
        return "planned"

    else:
        # Plan creation failed
        shared["planning_error"] = "Invalid plan format returned"
        shared["current_plan"] = None
        eprint("Plan creation failed - invalid format")
        return "planning_failed"

prep_async(shared) async

Enhanced preparation with goals-based planning support

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

async def prep_async(self, shared):
    """Enhanced preparation with goals-based planning support"""

    # Check if this is a goals-based call from LLMReasonerNode
    replan_context = shared.get("replan_context", {})
    goals_list = replan_context.get("goals", [])

    if goals_list:
        # Goals-based planning (called by LLMReasonerNode)
        return {
            "goals": goals_list,
            "planning_mode": "goals_based",
            "query": shared.get("current_query", ""),
            "reasoning_context": replan_context.get("reasoning_context", ""),
            "triggered_by": replan_context.get("triggered_by", "unknown"),
            "tasks": shared.get("tasks", {}),
            "system_status": shared.get("system_status", "idle"),
            "tool_capabilities": shared.get("tool_capabilities", {}),
            "available_tools_names": shared.get("available_tools", []),
            "strategy": "goals_decomposition",  # New strategy type
            "fast_llm_model": shared.get("fast_llm_model"),
            "complex_llm_model": shared.get("complex_llm_model"),
            "agent_instance": shared.get("agent_instance"),
            "variable_manager": shared.get("variable_manager"),
        }
    else:
        # Legacy planning (original query-based approach)
        return {
            "query": shared.get("current_query", ""),
            "tasks": shared.get("tasks", {}),
            "system_status": shared.get("system_status", "idle"),
            "tool_capabilities": shared.get("tool_capabilities", {}),
            "available_tools_names": shared.get("available_tools", []),
            "strategy": shared.get("selected_strategy", "direct_response"),
            "fast_llm_model": shared.get("fast_llm_model"),
            "complex_llm_model": shared.get("complex_llm_model"),
            "agent_instance": shared.get("agent_instance"),
            "variable_manager": shared.get("variable_manager"),
            "planning_mode": "legacy"
        }

ToolAnalysis

Bases: BaseModel

Defines the structure for a valid tool analysis.

Source code in toolboxv2/mods/isaa/base/Agent/types.py

class ToolAnalysis(BaseModel):
    """Defines the structure for a valid tool analysis."""
    primary_function: str = Field(..., description="The main purpose of the tool.")
    use_cases: list[str] = Field(..., description="Specific use cases for the tool.")
    trigger_phrases: list[str] = Field(..., description="Phrases that should trigger the tool.")
    indirect_connections: list[str] = Field(..., description="Non-obvious connections or applications.")
    complexity_scenarios: list[str] = Field(..., description="Complex scenarios where the tool can be applied.")
    user_intent_categories: list[str] = Field(..., description="Categories of user intent the tool addresses.")
    confidence_triggers: dict[str, float] = Field(..., description="Phrases mapped to confidence scores.")
    tool_complexity: str = Field(..., description="The complexity of the tool, rated as low, medium, or high.")
    args_schema: dict[str, Any] | None = Field(..., description="The schema for the tool's arguments.")

ToolTask dataclass

Bases: Task

Spezialisierter Task für Tool-Aufrufe

Source code in toolboxv2/mods/isaa/base/Agent/types.py

@dataclass
class ToolTask(Task):
    """Spezialisierter Task für Tool-Aufrufe"""
    tool_name: str = ""
    arguments: dict[str, Any] = field(default_factory=dict)  # Kann {{ }} Referenzen enthalten
    hypothesis: str = ""  # Was erwarten wir von diesem Tool?
    validation_criteria: str = ""  # Wie validieren wir das Ergebnis?
    expectation: str = ""  # Wie sollte das Ergebnis aussehen?

UnifiedBindingManager

Unified manager that handles both shared and private variable scopes for bound agents

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

class UnifiedBindingManager:
    """Unified manager that handles both shared and private variable scopes for bound agents"""

    def __init__(self, shared_manager: VariableManager, private_manager: VariableManager,
                 agent_name: str, shared_scopes: list[str], auto_sync: bool, binding_config: dict):
        self.shared_manager = shared_manager
        self.private_manager = private_manager
        self.agent_name = agent_name
        self.shared_scopes = shared_scopes
        self.auto_sync = auto_sync
        self.binding_config = binding_config

    def get(self, path: str, default=None, use_cache: bool = True):
        """Get variable from appropriate manager (shared or private)"""
        scope = path.split('.')[0] if '.' in path else path

        if scope in self.shared_scopes:
            return self.shared_manager.get(path, default, use_cache)
        else:
            # Try private first, then shared as fallback
            result = self.private_manager.get(path, None, use_cache)
            if result is None:
                return self.shared_manager.get(path, default, use_cache)
            return result

    def set(self, path: str, value, create_scope: bool = True):
        """Set variable in appropriate manager (shared or private)"""
        scope = path.split('.')[0] if '.' in path else path

        if scope in self.shared_scopes:
            self.shared_manager.set(path, value, create_scope)
            # Auto-sync to other bound agents if enabled
            if self.auto_sync:
                self._sync_to_bound_agents(path, value)
        else:
            # Private scope - add agent identifier
            private_path = f"{path}_{self.agent_name}" if not path.endswith(f"_{self.agent_name}") else path
            self.private_manager.set(private_path, value, create_scope)

    def _sync_to_bound_agents(self, path: str, value):
        """Sync shared variable changes to all bound agents"""
        try:
            bound_agents = self.binding_config.get('agents', [])
            for agent in bound_agents:
                if (agent.amd.name != self.agent_name and
                    hasattr(agent, 'variable_manager') and
                    isinstance(agent.variable_manager, UnifiedBindingManager)):
                    agent.variable_manager.shared_manager.set(path, value, create_scope=True)
        except Exception as e:
            wprint(f"Auto-sync failed for path {path}: {e}")

    def format_text(self, text: str, context: dict = None) -> str:
        """Format text with variables from both managers"""
        # First try private manager, then shared manager
        try:
            result = self.private_manager.format_text(text, context)
            return self.shared_manager.format_text(result, context)
        except:
            return self.shared_manager.format_text(text, context)

    def get_available_variables(self) -> dict[str, dict]:
        """Get available variables from both managers"""
        shared_vars = self.shared_manager.get_available_variables()
        private_vars = self.private_manager.get_available_variables()

        # Merge with prefix for private vars
        combined = shared_vars.copy()
        for key, value in private_vars.items():
            combined[f"private_{self.agent_name}_{key}"] = value

        return combined

    def get_scope_info(self) -> dict[str, Any]:
        """Get scope information from both managers"""
        shared_info = self.shared_manager.get_scope_info()
        private_info = self.private_manager.get_scope_info()

        return {
            'shared_scopes': shared_info,
            'private_scopes': private_info,
            'binding_info': {
                'agent_name': self.agent_name,
                'binding_id': self.binding_config.get('binding_id'),
                'auto_sync': self.auto_sync
            }
        }

    # Delegate other methods to shared manager by default
    def __getattr__(self, name):
        return getattr(self.shared_manager, name)

format_text(text, context=None)

Format text with variables from both managers

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

def format_text(self, text: str, context: dict = None) -> str:
    """Format text with variables from both managers"""
    # First try private manager, then shared manager
    try:
        result = self.private_manager.format_text(text, context)
        return self.shared_manager.format_text(result, context)
    except:
        return self.shared_manager.format_text(text, context)

get(path, default=None, use_cache=True)

Get variable from appropriate manager (shared or private)

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

def get(self, path: str, default=None, use_cache: bool = True):
    """Get variable from appropriate manager (shared or private)"""
    scope = path.split('.')[0] if '.' in path else path

    if scope in self.shared_scopes:
        return self.shared_manager.get(path, default, use_cache)
    else:
        # Try private first, then shared as fallback
        result = self.private_manager.get(path, None, use_cache)
        if result is None:
            return self.shared_manager.get(path, default, use_cache)
        return result

get_available_variables()

Get available variables from both managers

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

def get_available_variables(self) -> dict[str, dict]:
    """Get available variables from both managers"""
    shared_vars = self.shared_manager.get_available_variables()
    private_vars = self.private_manager.get_available_variables()

    # Merge with prefix for private vars
    combined = shared_vars.copy()
    for key, value in private_vars.items():
        combined[f"private_{self.agent_name}_{key}"] = value

    return combined

get_scope_info()

Get scope information from both managers

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

def get_scope_info(self) -> dict[str, Any]:
    """Get scope information from both managers"""
    shared_info = self.shared_manager.get_scope_info()
    private_info = self.private_manager.get_scope_info()

    return {
        'shared_scopes': shared_info,
        'private_scopes': private_info,
        'binding_info': {
            'agent_name': self.agent_name,
            'binding_id': self.binding_config.get('binding_id'),
            'auto_sync': self.auto_sync
        }
    }

set(path, value, create_scope=True)

Set variable in appropriate manager (shared or private)

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

def set(self, path: str, value, create_scope: bool = True):
    """Set variable in appropriate manager (shared or private)"""
    scope = path.split('.')[0] if '.' in path else path

    if scope in self.shared_scopes:
        self.shared_manager.set(path, value, create_scope)
        # Auto-sync to other bound agents if enabled
        if self.auto_sync:
            self._sync_to_bound_agents(path, value)
    else:
        # Private scope - add agent identifier
        private_path = f"{path}_{self.agent_name}" if not path.endswith(f"_{self.agent_name}") else path
        self.private_manager.set(private_path, value, create_scope)

UnifiedContextManager

Zentrale Orchestrierung aller Context-Quellen für einheitlichen und effizienten Datenzugriff. Vereinigt ChatSession, VariableManager, World Model und Task Results.

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

class UnifiedContextManager:
    """
    Zentrale Orchestrierung aller Context-Quellen für einheitlichen und effizienten Datenzugriff.
    Vereinigt ChatSession, VariableManager, World Model und Task Results.
    """

    def __init__(self, agent):
        self.agent = agent
        self.session_managers: dict[str, Any] = {}  # ChatSession objects
        self.variable_manager: VariableManager = None
        self.compression_threshold = 15  # Messages before compression
        self._context_cache: dict[str, tuple[float, Any]] = {}  # (timestamp, data)
        self.cache_ttl = 300  # 5 minutes
        self._memory_instance = None

    async def initialize_session(self, session_id: str, max_history: int = 200):
        """Initialisiere oder lade existierende ChatSession als primäre Context-Quelle"""
        if session_id not in self.session_managers:
            try:
                # Get memory instance
                if not self._memory_instance:
                    from toolboxv2 import get_app
                    self._memory_instance = get_app().get_mod("isaa").get_memory()
                from toolboxv2.mods.isaa.extras.session import ChatSession
                # Create ChatSession as PRIMARY memory source
                session = ChatSession(
                    self._memory_instance,
                    max_length=max_history,
                    space_name=f"ChatSession/{self.agent.amd.name}.{session_id}.unified"
                )
                self.session_managers[session_id] = session

                # Integration mit VariableManager wenn verfügbar
                if self.variable_manager:
                    self.variable_manager.register_scope(f'session_{session_id}', {
                        'chat_session_active': True,
                        'history_length': len(session.history),
                        'last_interaction': None,
                        'session_id': session_id
                    })

                rprint(f"Unified session context initialized for {session_id}")
                return session

            except Exception as e:
                eprint(f"Failed to create ChatSession for {session_id}: {e}")
                # Fallback: Create minimal session manager
                self.session_managers[session_id] = {
                    'history': [],
                    'session_id': session_id,
                    'fallback_mode': True
                }
                return self.session_managers[session_id]

        return self.session_managers[session_id]

    async def add_interaction(self, session_id: str, role: str, content: str, metadata: dict = None) -> None:
        """Einheitlicher Weg um Interaktionen in ChatSession zu speichern"""
        session = await self.initialize_session(session_id)

        message = {
            'role': role,
            'content': content,
            'timestamp': datetime.now().isoformat(),
            'session_id': session_id,
            'metadata': metadata or {}
        }

        # PRIMARY: Store in ChatSession
        if hasattr(session, 'add_message'):
            from toolboxv2 import get_app
            get_app().run_bg_task_advanced(session.add_message, message, direct=False)
        elif isinstance(session, dict) and 'history' in session:
            # Fallback mode
            session['history'].append(message)
            # Keep max length
            max_len = 200
            if len(session['history']) > max_len:
                session['history'] = session['history'][-max_len:]

        # SECONDARY: Update VariableManager
        if self.variable_manager:
            self.variable_manager.set(f'session_{session_id}.last_interaction', message)
            if hasattr(session, 'history'):
                self.variable_manager.set(f'session_{session_id}.history_length', len(session.history))
            elif isinstance(session, dict):
                self.variable_manager.set(f'session_{session_id}.history_length', len(session.get('history', [])))

        # Clear context cache for this session
        self._invalidate_cache(session_id)

    async def get_contextual_history(self, session_id: str, query: str = "", max_entries: int = 10) -> list[dict]:
        """Intelligente Auswahl relevanter Geschichte aus ChatSession"""
        session = self.session_managers.get(session_id)
        if not session:
            return []

        try:
            # ChatSession mode
            if hasattr(session, 'get_past_x'):
                recent_history = session.get_past_x(max_entries, last_u=False)
                c = await session.get_reference(query)
                return recent_history[:max_entries] + ([] if not c else  [{'role': 'system', 'content': c,
                                                        'timestamp': datetime.now().isoformat(), 'metadata': {'source': 'contextual_history'}}] )

            # Fallback mode
            elif isinstance(session, dict) and 'history' in session:
                history = session['history']
                # Return last max_entries, starting with last user message
                result = []
                for msg in reversed(history[-max_entries:]):
                    result.append(msg)
                    if msg.get('role') == 'user' and len(result) >= max_entries:
                        break
                return list(reversed(result))[:max_entries]

        except Exception as e:
            eprint(f"Error getting contextual history: {e}")

        return []

    async def build_unified_context(self, session_id: str, query: str = None, context_type: str = "full") -> dict[
        str, Any]:
        """ZENTRALE Methode für vollständigen Context-Aufbau aus allen Quellen"""

        # Cache check
        cache_key = f"{session_id}_{hash(query or '')}_{context_type}"
        cached = self._get_cached_context(cache_key)
        if cached:
            return cached

        context: dict[str, Any] = {
            'timestamp': datetime.now().isoformat(),
            'session_id': session_id,
            'query': query,
            'context_type': context_type
        }

        try:
            # 1. CHAT HISTORY (Primary - from ChatSession)
            with Spinner("Building unified context..."):
                context['chat_history'] = await self.get_contextual_history(
                    session_id, query or "", max_entries=15
                )

            # 2. VARIABLE SYSTEM STATE
            if self.variable_manager:
                context['variables'] = {
                    'available_scopes': list(self.variable_manager.scopes.keys()),
                    'total_variables': len(self.variable_manager.get_available_variables()),
                    'recent_results': self._get_recent_results(5)
                }
            else:
                context['variables'] = {'status': 'variable_manager_not_available'}

            # 3. WORLD MODEL FACTS
            if self.variable_manager:
                world_model = self.variable_manager.get('world', {})
                if world_model and query:
                    context['relevant_facts'] = self._extract_relevant_facts(world_model, query)
                else:
                    context['relevant_facts'] = list(world_model.items())[:5]  # Top 5 facts

            # 4. EXECUTION STATE
            context['execution_state'] = {
                'active_tasks': self._get_active_tasks(),
                'recent_completions': self._get_recent_completions(3),
                'system_status': self.agent.shared.get('system_status', 'idle')
            }

            # 5. SESSION STATISTICS
            context['session_stats'] = {
                'total_sessions': len(self.session_managers),
                'current_session_length': len(context['chat_history']),
                'cache_enabled': bool(self._context_cache)
            }

        except Exception as e:
            eprint(f"Error building unified context: {e}")
            context['error'] = str(e)
            context['fallback_mode'] = True

        # Cache result
        self._cache_context(cache_key, context)
        return context

    def get_formatted_context_for_llm(self, unified_context: dict[str, Any]) -> str:
        """Formatiere unified context für LLM consumption"""
        try:
            parts = []

            # Header with session info
            session_id = unified_context.get('session_id', 'unknown')
            query = unified_context.get('query', '')
            context_type = unified_context.get('context_type', 'full')
            parts.append(f"## Session Context ({context_type})")
            parts.append(f"Session: {session_id}")
            if query:
                parts.append(f"Query: {query}")

            # Recent Chat History
            chat_history = unified_context.get('chat_history', [])
            if chat_history:
                parts.append("\n## Recent Conversation")
                for msg in chat_history[-5:]:  # Last 5 messages
                    timestamp = msg.get('timestamp', '')[:19]  # Remove microseconds
                    role = msg.get('role', 'unknown')
                    content = msg.get('content', '')
                    content_preview = content[:500] + ("..." if len(content) > 500 else "")
                    parts.append(f"[{timestamp}] {role}: {content_preview}")

            # Variable System State
            variables = unified_context.get('variables', {})
            if variables and variables != {'status': 'variable_manager_not_available'}:
                parts.append("\n## Variable System")

                # Available scopes
                scopes = variables.get('available_scopes', [])
                if scopes:
                    parts.append(f"Available Scopes: {', '.join(scopes)}")

                # Total variables count
                total_vars = variables.get('total_variables', 0)
                if total_vars > 0:
                    parts.append(f"Total Variables: {total_vars}")
                    parts.append(f"Total Variables Values: \n{yaml.dump(self.variable_manager.get_available_variables())}")

                # Recent results
                recent_results = variables.get('recent_results', [])
                if recent_results:
                    parts.append(f"Recent Results ({len(recent_results)}):")
                    for result in recent_results[:3]:  # Top 3 results
                        task_id = result.get('task_id', 'unknown')
                        preview = str(result.get('preview', ''))[:100]
                        preview += "..." if len(str(result.get('preview', ''))) > 100 else ""
                        parts.append(f"  - {task_id}: {preview}")
            elif variables.get('status') == 'variable_manager_not_available':
                parts.append("\n## Variable System: Not Available")

            # World Model Facts (Relevant Facts)
            relevant_facts = unified_context.get('relevant_facts', [])
            if relevant_facts:
                parts.append("\n## World Model Facts")
                for fact in relevant_facts[:5]:  # Top 5 facts
                    if isinstance(fact, (list, tuple)) and len(fact) >= 2:
                        # Handle (key, value) tuple format
                        key, value = fact[0], fact[1]
                        fact_preview = str(value)[:100] + ("..." if len(str(value)) > 100 else "")
                        parts.append(f"- {key}: {fact_preview}")
                    elif isinstance(fact, dict):
                        # Handle dict format
                        for key, value in list(fact.items())[:1]:  # Just first item
                            fact_preview = str(value)[:100] + ("..." if len(str(value)) > 100 else "")
                            parts.append(f"- {key}: {fact_preview}")

            # Execution State
            execution_state = unified_context.get('execution_state', {})
            if execution_state:
                parts.append("\n## System Status")

                system_status = execution_state.get('system_status', 'unknown')
                parts.append(f"Status: {system_status}")

                active_tasks = execution_state.get('active_tasks', [])
                if active_tasks:
                    parts.append(f"Active Tasks: {len(active_tasks)}")
                    # Show task previews if available
                    for task in active_tasks[:2]:  # Show first 2 tasks
                        task_str = str(task)[:80] + ("..." if len(str(task)) > 80 else "")
                        parts.append(f"  - {task_str}")

                recent_completions = execution_state.get('recent_completions', [])
                if recent_completions:
                    parts.append(f"Recent Completions: {len(recent_completions)}")
                    # Show completion previews if available
                    for completion in recent_completions[:2]:  # Show first 2 completions
                        comp_str = str(completion)[:80] + ("..." if len(str(completion)) > 80 else "")
                        parts.append(f"  - {comp_str}")

            # Session Statistics
            session_stats = unified_context.get('session_stats', {})
            if session_stats:
                parts.append("\n## Session Statistics")

                total_sessions = session_stats.get('total_sessions', 0)
                if total_sessions > 0:
                    parts.append(f"Total Sessions: {total_sessions}")

                current_length = session_stats.get('current_session_length', 0)
                if current_length > 0:
                    parts.append(f"Current Session Length: {current_length} messages")

                cache_enabled = session_stats.get('cache_enabled', False)
                parts.append(f"Cache Enabled: {cache_enabled}")

            # Error handling
            if unified_context.get('error'):
                parts.append(f"\n## Error")
                parts.append(f"Error: {unified_context['error']}")

            if unified_context.get('fallback_mode'):
                parts.append("⚠️  Running in fallback mode")

            # Footer with timestamp
            timestamp = unified_context.get('timestamp', 'unknown')
            parts.append(f"\n---\nContext generated at: {timestamp}")

            return "\n".join(parts)

        except Exception as e:
            eprint(f"Error formatting context for LLM: {e}")
            import traceback
            print(traceback.format_exc())
            return f"Context formatting error: {str(e)}"

    def _merge_and_dedupe_history(self, recent_history: list[dict], relevant_refs: list) -> list[dict]:
        """Merge und dedupliziere History-Einträge"""
        try:
            merged = recent_history.copy()

            # Add relevant references if they're not already in recent history
            for ref in relevant_refs:
                # Convert ref to message format if needed
                if isinstance(ref, dict) and 'content' in ref:
                    # Check if not already in recent_history
                    is_duplicate = any(
                        msg.get('content', '') == ref.get('content', '') and
                        msg.get('timestamp', '') == ref.get('timestamp', '')
                        for msg in merged
                    )
                    if not is_duplicate:
                        merged.append(ref)

            # Sort by timestamp
            merged.sort(key=lambda x: x.get('timestamp', ''))

            return merged
        except:
            return recent_history

    def _get_recent_results(self, limit: int = 5) -> list[dict]:
        """Hole recent results aus dem shared state"""
        try:
            results_store = self.agent.shared.get("results", {})
            recent_results = []

            for task_id, result_data in list(results_store.items())[-limit:]:
                if result_data and result_data.get("data"):
                    preview = str(result_data["data"])[:150] + "..."
                    recent_results.append({
                        "task_id": task_id,
                        "preview": preview,
                        "success": result_data.get("metadata", {}).get("success", False),
                        "timestamp": result_data.get("metadata", {}).get("completed_at")
                    })

            return recent_results
        except:
            return []

    def _extract_relevant_facts(self, world_model: dict, query: str) -> list[tuple[str, Any]]:
        """Extrahiere relevante Facts basierend auf Query"""
        try:
            query_words = set(query.lower().split())
            relevant_facts = []

            for key, value in world_model.items():
                # Simple relevance scoring
                key_words = set(key.lower().split())
                value_words = set(str(value).lower().split())

                # Check for word overlap
                key_overlap = len(query_words.intersection(key_words))
                value_overlap = len(query_words.intersection(value_words))

                if key_overlap > 0 or value_overlap > 0:
                    relevance_score = key_overlap * 2 + value_overlap  # Key matches weighted higher
                    relevant_facts.append((relevance_score, key, value))

            # Sort by relevance and return top facts
            relevant_facts.sort(key=lambda x: x[0], reverse=True)
            return [(key, value) for _, key, value in relevant_facts[:5]]
        except:
            return list(world_model.items())[:5]

    def _get_active_tasks(self) -> list[dict]:
        """Hole aktive Tasks"""
        try:
            tasks = self.agent.shared.get("tasks", {})
            return [
                {"id": task_id, "description": task.description, "status": task.status}
                for task_id, task in tasks.items()
                if task.status == "running"
            ]
        except:
            return []

    def _get_recent_completions(self, limit: int = 3) -> list[dict]:
        """Hole recent completions"""
        try:
            tasks = self.agent.shared.get("tasks", {})
            completed = [
                {"id": task_id, "description": task.description, "completed_at": task.completed_at}
                for task_id, task in tasks.items()
                if task.status == "completed" and hasattr(task, 'completed_at') and task.completed_at
            ]
            # Sort by completion time
            completed.sort(key=lambda x: x.get('completed_at', ''), reverse=True)
            return completed[:limit]
        except:
            return []

    def _get_cached_context(self, cache_key: str) -> dict[str, Any] | None:
        """Hole Context aus Cache wenn noch gültig"""
        if cache_key in self._context_cache:
            timestamp, data = self._context_cache[cache_key]
            if time.time() - timestamp < self.cache_ttl:
                return data
            else:
                del self._context_cache[cache_key]
        return None

    def _cache_context(self, cache_key: str, context: dict[str, Any]):
        """Speichere Context in Cache"""
        self._context_cache[cache_key] = (time.time(), context.copy())

        # Cleanup old cache entries
        if len(self._context_cache) > 50:  # Keep max 50 entries
            oldest_key = min(self._context_cache.keys(),
                             key=lambda k: self._context_cache[k][0])
            del self._context_cache[oldest_key]

    def _invalidate_cache(self, session_id: str = None):
        """Invalidate cache for specific session or all"""
        if session_id:
            # Remove all cache entries for this session
            keys_to_remove = [k for k in self._context_cache if session_id in k]
            for key in keys_to_remove:
                del self._context_cache[key]
        else:
            self._context_cache.clear()

    def get_session_statistics(self) -> dict[str, Any]:
        """Hole Statistiken über alle Sessions"""
        stats = {
            "total_sessions": len(self.session_managers),
            "active_sessions": [],
            "cache_entries": len(self._context_cache),
            "cache_hit_rate": 0.0  # Could be tracked if needed
        }

        for session_id, session in self.session_managers.items():
            session_info = {
                "session_id": session_id,
                "fallback_mode": isinstance(session, dict) and session.get('fallback_mode', False)
            }

            if hasattr(session, 'history'):
                session_info["message_count"] = len(session.history)
            elif isinstance(session, dict) and 'history' in session:
                session_info["message_count"] = len(session['history'])

            stats["active_sessions"].append(session_info)

        return stats

    async def cleanup_old_sessions(self, max_age_hours: int = 168) -> int:
        """Cleanup alte Sessions (default: 1 Woche)"""
        try:
            cutoff_time = datetime.now() - timedelta(hours=max_age_hours)
            removed_count = 0

            sessions_to_remove = []
            for session_id, session in self.session_managers.items():
                should_remove = False

                # Check last activity
                if hasattr(session, 'history') and session.history:
                    last_msg = session.history[-1]
                    last_timestamp = last_msg.get('timestamp')
                    if last_timestamp:
                        try:
                            last_time = datetime.fromisoformat(last_timestamp.replace('Z', '+00:00'))
                            if last_time < cutoff_time:
                                should_remove = True
                        except:
                            pass
                elif isinstance(session, dict) and session.get('history'):
                    last_msg = session['history'][-1]
                    last_timestamp = last_msg.get('timestamp')
                    if last_timestamp:
                        try:
                            last_time = datetime.fromisoformat(last_timestamp.replace('Z', '+00:00'))
                            if last_time < cutoff_time:
                                should_remove = True
                        except:
                            pass

                if should_remove:
                    sessions_to_remove.append(session_id)

            # Remove old sessions
            for session_id in sessions_to_remove:
                session = self.session_managers[session_id]
                if hasattr(session, 'on_exit'):
                    session.on_exit()  # Save ChatSession data
                del self.session_managers[session_id]
                removed_count += 1

                # Remove from variable manager
                if self.variable_manager:
                    scope_name = f'session_{session_id}'
                    if scope_name in self.variable_manager.scopes:
                        del self.variable_manager.scopes[scope_name]

            # Clear related cache entries
            self._invalidate_cache()

            return removed_count
        except Exception as e:
            eprint(f"Error cleaning up old sessions: {e}")
            return 0

add_interaction(session_id, role, content, metadata=None) async

Einheitlicher Weg um Interaktionen in ChatSession zu speichern

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

async def add_interaction(self, session_id: str, role: str, content: str, metadata: dict = None) -> None:
    """Einheitlicher Weg um Interaktionen in ChatSession zu speichern"""
    session = await self.initialize_session(session_id)

    message = {
        'role': role,
        'content': content,
        'timestamp': datetime.now().isoformat(),
        'session_id': session_id,
        'metadata': metadata or {}
    }

    # PRIMARY: Store in ChatSession
    if hasattr(session, 'add_message'):
        from toolboxv2 import get_app
        get_app().run_bg_task_advanced(session.add_message, message, direct=False)
    elif isinstance(session, dict) and 'history' in session:
        # Fallback mode
        session['history'].append(message)
        # Keep max length
        max_len = 200
        if len(session['history']) > max_len:
            session['history'] = session['history'][-max_len:]

    # SECONDARY: Update VariableManager
    if self.variable_manager:
        self.variable_manager.set(f'session_{session_id}.last_interaction', message)
        if hasattr(session, 'history'):
            self.variable_manager.set(f'session_{session_id}.history_length', len(session.history))
        elif isinstance(session, dict):
            self.variable_manager.set(f'session_{session_id}.history_length', len(session.get('history', [])))

    # Clear context cache for this session
    self._invalidate_cache(session_id)

build_unified_context(session_id, query=None, context_type='full') async

ZENTRALE Methode für vollständigen Context-Aufbau aus allen Quellen

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

async def build_unified_context(self, session_id: str, query: str = None, context_type: str = "full") -> dict[
    str, Any]:
    """ZENTRALE Methode für vollständigen Context-Aufbau aus allen Quellen"""

    # Cache check
    cache_key = f"{session_id}_{hash(query or '')}_{context_type}"
    cached = self._get_cached_context(cache_key)
    if cached:
        return cached

    context: dict[str, Any] = {
        'timestamp': datetime.now().isoformat(),
        'session_id': session_id,
        'query': query,
        'context_type': context_type
    }

    try:
        # 1. CHAT HISTORY (Primary - from ChatSession)
        with Spinner("Building unified context..."):
            context['chat_history'] = await self.get_contextual_history(
                session_id, query or "", max_entries=15
            )

        # 2. VARIABLE SYSTEM STATE
        if self.variable_manager:
            context['variables'] = {
                'available_scopes': list(self.variable_manager.scopes.keys()),
                'total_variables': len(self.variable_manager.get_available_variables()),
                'recent_results': self._get_recent_results(5)
            }
        else:
            context['variables'] = {'status': 'variable_manager_not_available'}

        # 3. WORLD MODEL FACTS
        if self.variable_manager:
            world_model = self.variable_manager.get('world', {})
            if world_model and query:
                context['relevant_facts'] = self._extract_relevant_facts(world_model, query)
            else:
                context['relevant_facts'] = list(world_model.items())[:5]  # Top 5 facts

        # 4. EXECUTION STATE
        context['execution_state'] = {
            'active_tasks': self._get_active_tasks(),
            'recent_completions': self._get_recent_completions(3),
            'system_status': self.agent.shared.get('system_status', 'idle')
        }

        # 5. SESSION STATISTICS
        context['session_stats'] = {
            'total_sessions': len(self.session_managers),
            'current_session_length': len(context['chat_history']),
            'cache_enabled': bool(self._context_cache)
        }

    except Exception as e:
        eprint(f"Error building unified context: {e}")
        context['error'] = str(e)
        context['fallback_mode'] = True

    # Cache result
    self._cache_context(cache_key, context)
    return context

cleanup_old_sessions(max_age_hours=168) async

Cleanup alte Sessions (default: 1 Woche)

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

async def cleanup_old_sessions(self, max_age_hours: int = 168) -> int:
    """Cleanup alte Sessions (default: 1 Woche)"""
    try:
        cutoff_time = datetime.now() - timedelta(hours=max_age_hours)
        removed_count = 0

        sessions_to_remove = []
        for session_id, session in self.session_managers.items():
            should_remove = False

            # Check last activity
            if hasattr(session, 'history') and session.history:
                last_msg = session.history[-1]
                last_timestamp = last_msg.get('timestamp')
                if last_timestamp:
                    try:
                        last_time = datetime.fromisoformat(last_timestamp.replace('Z', '+00:00'))
                        if last_time < cutoff_time:
                            should_remove = True
                    except:
                        pass
            elif isinstance(session, dict) and session.get('history'):
                last_msg = session['history'][-1]
                last_timestamp = last_msg.get('timestamp')
                if last_timestamp:
                    try:
                        last_time = datetime.fromisoformat(last_timestamp.replace('Z', '+00:00'))
                        if last_time < cutoff_time:
                            should_remove = True
                    except:
                        pass

            if should_remove:
                sessions_to_remove.append(session_id)

        # Remove old sessions
        for session_id in sessions_to_remove:
            session = self.session_managers[session_id]
            if hasattr(session, 'on_exit'):
                session.on_exit()  # Save ChatSession data
            del self.session_managers[session_id]
            removed_count += 1

            # Remove from variable manager
            if self.variable_manager:
                scope_name = f'session_{session_id}'
                if scope_name in self.variable_manager.scopes:
                    del self.variable_manager.scopes[scope_name]

        # Clear related cache entries
        self._invalidate_cache()

        return removed_count
    except Exception as e:
        eprint(f"Error cleaning up old sessions: {e}")
        return 0

get_contextual_history(session_id, query='', max_entries=10) async

Intelligente Auswahl relevanter Geschichte aus ChatSession

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

async def get_contextual_history(self, session_id: str, query: str = "", max_entries: int = 10) -> list[dict]:
    """Intelligente Auswahl relevanter Geschichte aus ChatSession"""
    session = self.session_managers.get(session_id)
    if not session:
        return []

    try:
        # ChatSession mode
        if hasattr(session, 'get_past_x'):
            recent_history = session.get_past_x(max_entries, last_u=False)
            c = await session.get_reference(query)
            return recent_history[:max_entries] + ([] if not c else  [{'role': 'system', 'content': c,
                                                    'timestamp': datetime.now().isoformat(), 'metadata': {'source': 'contextual_history'}}] )

        # Fallback mode
        elif isinstance(session, dict) and 'history' in session:
            history = session['history']
            # Return last max_entries, starting with last user message
            result = []
            for msg in reversed(history[-max_entries:]):
                result.append(msg)
                if msg.get('role') == 'user' and len(result) >= max_entries:
                    break
            return list(reversed(result))[:max_entries]

    except Exception as e:
        eprint(f"Error getting contextual history: {e}")

    return []

get_formatted_context_for_llm(unified_context)

Formatiere unified context für LLM consumption

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

def get_formatted_context_for_llm(self, unified_context: dict[str, Any]) -> str:
    """Formatiere unified context für LLM consumption"""
    try:
        parts = []

        # Header with session info
        session_id = unified_context.get('session_id', 'unknown')
        query = unified_context.get('query', '')
        context_type = unified_context.get('context_type', 'full')
        parts.append(f"## Session Context ({context_type})")
        parts.append(f"Session: {session_id}")
        if query:
            parts.append(f"Query: {query}")

        # Recent Chat History
        chat_history = unified_context.get('chat_history', [])
        if chat_history:
            parts.append("\n## Recent Conversation")
            for msg in chat_history[-5:]:  # Last 5 messages
                timestamp = msg.get('timestamp', '')[:19]  # Remove microseconds
                role = msg.get('role', 'unknown')
                content = msg.get('content', '')
                content_preview = content[:500] + ("..." if len(content) > 500 else "")
                parts.append(f"[{timestamp}] {role}: {content_preview}")

        # Variable System State
        variables = unified_context.get('variables', {})
        if variables and variables != {'status': 'variable_manager_not_available'}:
            parts.append("\n## Variable System")

            # Available scopes
            scopes = variables.get('available_scopes', [])
            if scopes:
                parts.append(f"Available Scopes: {', '.join(scopes)}")

            # Total variables count
            total_vars = variables.get('total_variables', 0)
            if total_vars > 0:
                parts.append(f"Total Variables: {total_vars}")
                parts.append(f"Total Variables Values: \n{yaml.dump(self.variable_manager.get_available_variables())}")

            # Recent results
            recent_results = variables.get('recent_results', [])
            if recent_results:
                parts.append(f"Recent Results ({len(recent_results)}):")
                for result in recent_results[:3]:  # Top 3 results
                    task_id = result.get('task_id', 'unknown')
                    preview = str(result.get('preview', ''))[:100]
                    preview += "..." if len(str(result.get('preview', ''))) > 100 else ""
                    parts.append(f"  - {task_id}: {preview}")
        elif variables.get('status') == 'variable_manager_not_available':
            parts.append("\n## Variable System: Not Available")

        # World Model Facts (Relevant Facts)
        relevant_facts = unified_context.get('relevant_facts', [])
        if relevant_facts:
            parts.append("\n## World Model Facts")
            for fact in relevant_facts[:5]:  # Top 5 facts
                if isinstance(fact, (list, tuple)) and len(fact) >= 2:
                    # Handle (key, value) tuple format
                    key, value = fact[0], fact[1]
                    fact_preview = str(value)[:100] + ("..." if len(str(value)) > 100 else "")
                    parts.append(f"- {key}: {fact_preview}")
                elif isinstance(fact, dict):
                    # Handle dict format
                    for key, value in list(fact.items())[:1]:  # Just first item
                        fact_preview = str(value)[:100] + ("..." if len(str(value)) > 100 else "")
                        parts.append(f"- {key}: {fact_preview}")

        # Execution State
        execution_state = unified_context.get('execution_state', {})
        if execution_state:
            parts.append("\n## System Status")

            system_status = execution_state.get('system_status', 'unknown')
            parts.append(f"Status: {system_status}")

            active_tasks = execution_state.get('active_tasks', [])
            if active_tasks:
                parts.append(f"Active Tasks: {len(active_tasks)}")
                # Show task previews if available
                for task in active_tasks[:2]:  # Show first 2 tasks
                    task_str = str(task)[:80] + ("..." if len(str(task)) > 80 else "")
                    parts.append(f"  - {task_str}")

            recent_completions = execution_state.get('recent_completions', [])
            if recent_completions:
                parts.append(f"Recent Completions: {len(recent_completions)}")
                # Show completion previews if available
                for completion in recent_completions[:2]:  # Show first 2 completions
                    comp_str = str(completion)[:80] + ("..." if len(str(completion)) > 80 else "")
                    parts.append(f"  - {comp_str}")

        # Session Statistics
        session_stats = unified_context.get('session_stats', {})
        if session_stats:
            parts.append("\n## Session Statistics")

            total_sessions = session_stats.get('total_sessions', 0)
            if total_sessions > 0:
                parts.append(f"Total Sessions: {total_sessions}")

            current_length = session_stats.get('current_session_length', 0)
            if current_length > 0:
                parts.append(f"Current Session Length: {current_length} messages")

            cache_enabled = session_stats.get('cache_enabled', False)
            parts.append(f"Cache Enabled: {cache_enabled}")

        # Error handling
        if unified_context.get('error'):
            parts.append(f"\n## Error")
            parts.append(f"Error: {unified_context['error']}")

        if unified_context.get('fallback_mode'):
            parts.append("⚠️  Running in fallback mode")

        # Footer with timestamp
        timestamp = unified_context.get('timestamp', 'unknown')
        parts.append(f"\n---\nContext generated at: {timestamp}")

        return "\n".join(parts)

    except Exception as e:
        eprint(f"Error formatting context for LLM: {e}")
        import traceback
        print(traceback.format_exc())
        return f"Context formatting error: {str(e)}"

get_session_statistics()

Hole Statistiken über alle Sessions

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

def get_session_statistics(self) -> dict[str, Any]:
    """Hole Statistiken über alle Sessions"""
    stats = {
        "total_sessions": len(self.session_managers),
        "active_sessions": [],
        "cache_entries": len(self._context_cache),
        "cache_hit_rate": 0.0  # Could be tracked if needed
    }

    for session_id, session in self.session_managers.items():
        session_info = {
            "session_id": session_id,
            "fallback_mode": isinstance(session, dict) and session.get('fallback_mode', False)
        }

        if hasattr(session, 'history'):
            session_info["message_count"] = len(session.history)
        elif isinstance(session, dict) and 'history' in session:
            session_info["message_count"] = len(session['history'])

        stats["active_sessions"].append(session_info)

    return stats

initialize_session(session_id, max_history=200) async

Initialisiere oder lade existierende ChatSession als primäre Context-Quelle

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

async def initialize_session(self, session_id: str, max_history: int = 200):
    """Initialisiere oder lade existierende ChatSession als primäre Context-Quelle"""
    if session_id not in self.session_managers:
        try:
            # Get memory instance
            if not self._memory_instance:
                from toolboxv2 import get_app
                self._memory_instance = get_app().get_mod("isaa").get_memory()
            from toolboxv2.mods.isaa.extras.session import ChatSession
            # Create ChatSession as PRIMARY memory source
            session = ChatSession(
                self._memory_instance,
                max_length=max_history,
                space_name=f"ChatSession/{self.agent.amd.name}.{session_id}.unified"
            )
            self.session_managers[session_id] = session

            # Integration mit VariableManager wenn verfügbar
            if self.variable_manager:
                self.variable_manager.register_scope(f'session_{session_id}', {
                    'chat_session_active': True,
                    'history_length': len(session.history),
                    'last_interaction': None,
                    'session_id': session_id
                })

            rprint(f"Unified session context initialized for {session_id}")
            return session

        except Exception as e:
            eprint(f"Failed to create ChatSession for {session_id}: {e}")
            # Fallback: Create minimal session manager
            self.session_managers[session_id] = {
                'history': [],
                'session_id': session_id,
                'fallback_mode': True
            }
            return self.session_managers[session_id]

    return self.session_managers[session_id]

VariableManager

Unified variable management system with advanced features

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

class VariableManager:
    """Unified variable management system with advanced features"""

    def __init__(self, world_model: dict, shared_state: dict = None):
        self.world_model = world_model
        self.shared_state = shared_state or {}
        self.scopes = {
            'world': world_model,
            'shared': self.shared_state,
            'results': {},
            'tasks': {},
            'user': {},
            'system': {}
        }
        self._cache = {}

    def register_scope(self, name: str, data: dict):
        """Register a new variable scope"""
        self.scopes[name] = data
        self._cache.clear()

    def set_results_store(self, results_store: dict):
        """Set the results store for task result references"""
        self.scopes['results'] = results_store
        self._cache.clear()

    def set_tasks_store(self, tasks_store: dict):
        """Set tasks store for task metadata access"""
        self.scopes['tasks'] = tasks_store
        self._cache.clear()

    def _resolve_path(self, path: str):
        """
        Internal helper to navigate a path that can contain both
        dictionary keys and list indices.
        """
        parts = path.split('.')

        # Determine the starting point
        if len(parts) == 1:
            # Simple key in the top-level world_model
            current = self.world_model
        else:
            scope_name = parts[0]
            if scope_name not in self.scopes:
                raise KeyError(f"Scope '{scope_name}' not found")
            current = self.scopes[scope_name]
            parts = parts[1:]  # Continue with the rest of the path

        # Navigate through the parts
        for part in parts:
            if isinstance(current, list):
                try:
                    # It's a list, so the part must be an integer index
                    index = int(part)
                    current = current[index]
                except (ValueError, IndexError):
                    raise KeyError(f"Invalid list index '{part}' in path '{path}'")
            elif isinstance(current, dict):
                try:
                    # It's a dictionary, so the part is a key
                    current = current[part]
                except KeyError:
                    raise KeyError(f"Key '{part}' not found in path '{path}'")
            else:
                # We've hit a non-collection type (int, str, etc.) but the path continues
                raise KeyError(f"Path cannot descend into non-collection type at '{part}' in path '{path}'")

        return current

    def get(self, path: str, default=None, use_cache: bool = True):
        """Get variable with dot notation path support for dicts and lists."""
        if use_cache and path in self._cache:
            return self._cache[path]

        try:
            value = self._resolve_path(path)
            if use_cache:
                self._cache[path] = value
            return value
        except (KeyError, IndexError):
            # A KeyError or IndexError during resolution means the path is invalid
            return default

    def set(self, path: str, value, create_scope: bool = True):
        """Set variable with dot notation path support for dicts and lists."""
        # Invalidate cache for this path
        if path in self._cache:
            del self._cache[path]

        parts = path.split('.')

        if len(parts) == 1:
            # Simple key in world_model
            self.world_model[path] = value
            return

        scope_name = parts[0]
        if scope_name not in self.scopes:
            if create_scope:
                self.scopes[scope_name] = {}
            else:
                raise KeyError(f"Scope '{scope_name}' not found")

        current = self.scopes[scope_name]

        # Iterate to the second-to-last part to get the container
        for i, part in enumerate(parts[1:-1]):
            next_part = parts[i + 2]  # Look ahead to the next part in the path

            # Determine if the current part is a dictionary key or a list index
            try:
                # Try to treat it as a list index
                key = int(part)
                if not isinstance(current, list):
                    # If current is not a list, we can't use an integer index
                    raise TypeError(f"Attempted to use integer index '{key}' on non-list for path '{path}'")

                # Ensure list is long enough
                while len(current) <= key:
                    current.append(None)  # Pad with None

                # If the next level doesn't exist, create it based on the next part
                if current[key] is None:
                    current[key] = [] if next_part.isdigit() else {}

                current = current[key]

            except ValueError:
                # It's a dictionary key
                key = part
                if not isinstance(current, dict):
                    raise TypeError(f"Attempted to use string key '{key}' on non-dict for path '{path}'")

                if key not in current:
                    # Create the next level: a list if the next part is a number, else a dict
                    current[key] = [] if next_part.isdigit() else {}

                current = current[key]

        # Handle the final part (the actual assignment)
        last_part = parts[-1]

        if isinstance(current, list):
            try:
                key = int(last_part)
                while len(current) <= key:
                    current.append(None)
                current[key] = value
            except ValueError:
                current.append(value)
        elif isinstance(current, dict):
            current[last_part] = value
        elif scope_name == 'tasks' and hasattr(current, 'task_identification_attr'):# from tasks like Tooltask ... model dump and acces
            dict_data = asdict(current)
            dict_data[last_part] = value
            current = dict_data
            # update self.scopes['tasks'] with the updated task
            self.scopes['tasks'][parts[1]][last_part] = current
        else:
            raise TypeError(f"Final container is not a list or dictionary for path '{path}' its a {type(current)}")

        self._cache.clear()

    def format_text(self, text: str, context: dict = None) -> str:
        """Enhanced text formatting with multiple syntaxes"""
        if not text or not isinstance(text, str):
            return str(text) if text is not None else ""

        # Temporary context overlay
        if context:
            original_scopes = self.scopes.copy()
            self.scopes['context'] = context

        try:
            # Handle {{ variable }} syntax
            formatted = self._format_double_braces(text)

            # Handle {variable} syntax
            formatted = self._format_single_braces(formatted)

            # Handle $variable syntax
            formatted = self._format_dollar_syntax(formatted)

            return formatted

        finally:
            if context:
                self.scopes = original_scopes

    def _format_double_braces(self, text: str) -> str:
        """Handle {{ variable.path }} syntax with improved debugging"""
        import re

        def replace_var(match):
            var_path = match.group(1).strip()
            value = self.get(var_path)

            if value is None:
                # IMPROVED: Log missing variables for debugging
                available_vars = list(self.get_available_variables().keys())
                wprint(f"Variable '{var_path}' not found. Available: {available_vars[:10]}")
                return match.group(0)  # Keep original if not found

            return self._value_to_string(value)

        return re.sub(r'\{\{\s*([^}]+)\s*\}\}', replace_var, text)

    def _format_single_braces(self, text: str) -> str:
        """Handle {variable.path} syntax, including with spaces like { variable.path }."""
        import re

        def replace_var(match):
            # Extrahiert den Variablennamen und entfernt führende/nachfolgende Leerzeichen
            var_path = match.group(1).strip()

            # Ruft den Wert über die get-Methode ab, die die Punktnotation bereits verarbeitet
            value = self.get(var_path)

            # Gibt den konvertierten Wert oder das Original-Tag zurück, wenn der Wert nicht gefunden wurde
            return self._value_to_string(value) if value is not None else match.group(0)

        # Dieser Regex findet {beliebiger.inhalt} und erlaubt Leerzeichen um den Inhalt
        # Er schließt verschachtelte oder leere Klammern wie {} oder { {var} } aus.
        return re.sub(r'\{([^{}]+)\}', replace_var, text)

    def _format_dollar_syntax(self, text: str) -> str:
        """Handle $variable syntax"""
        import re

        def replace_var(match):
            var_name = match.group(1)
            value = self.get(var_name)
            return self._value_to_string(value) if value is not None else match.group(0)

        return re.sub(r'\$([a-zA-Z_][a-zA-Z0-9_]*)', replace_var, text)

    def _value_to_string(self, value) -> str:
        """Convert value to string representation"""
        if isinstance(value, str):
            return value
        elif isinstance(value, dict | list):
            return json.dumps(value, default=str)
        else:
            return str(value)

    def validate_references(self, text: str) -> dict[str, bool]:
        """Validate all variable references in text"""
        import re

        references = {}

        # Find all {{ }} references
        double_brace_refs = re.findall(r'\{\{\s*([^}]+)\s*\}\}', text)
        for ref in double_brace_refs:
            references["{{"+ref+"}}"] = self.get(ref.strip()) is not None

        # Find all {} references
        single_brace_refs = re.findall(r'\{([^{}\s]+)\}', text)
        for ref in single_brace_refs:
            if '.' not in ref:  # Only simple vars
                references["{"+ref+"}"] = self.get(ref.strip()) is not None

        # Find all $ references
        dollar_refs = re.findall(r'\$([a-zA-Z_][a-zA-Z0-9_]*)', text)
        for ref in dollar_refs:
            references[f"${ref}"] = self.get(ref) is not None

        return references

    def get_scope_info(self) -> dict[str, Any]:
        """Get information about all available scopes"""
        info = {}
        for scope_name, scope_data in self.scopes.items():
            if isinstance(scope_data, dict):
                info[scope_name] = {
                    'type': 'dict',
                    'keys': len(scope_data),
                    'sample_keys': list(scope_data.keys())[:5]
                }
            else:
                info[scope_name] = {
                    'type': type(scope_data).__name__,
                    'value': str(scope_data)[:100]
                }
        return info

    def _validate_task_references(self, task: Task) -> dict[str, Any]:
        """Validate all variable references in a task"""
        validation_results = {
            'valid': True,
            'errors': [],
            'warnings': []
        }

        # Check different task types
        if isinstance(task, LLMTask):
            if task.prompt_template:
                refs = self.validate_references(task.prompt_template)
                for ref, is_valid in refs.items():
                    if not is_valid:
                        validation_results['errors'].append(f"Invalid reference in prompt: {ref}")
                        validation_results['valid'] = False

        elif isinstance(task, ToolTask):
            for key, value in task.arguments.items():
                if isinstance(value, str):
                    refs = self.validate_references(value)
                    for ref, is_valid in refs.items():
                        if not is_valid:
                            validation_results['warnings'].append(f"Invalid reference in {key}: {ref}")

        return validation_results

    def get_variable_suggestions(self, query: str) -> list[str]:
        """Get variable suggestions based on query content"""

        query_lower = query.lower()
        suggestions = []

        # Check all variables for relevance
        for scope in self.scopes.values():
            for name, var_def in scope.items():
                if name in ["system_context", "task_executor_instance",
                            "index", "tool_capabilities", "use_fast_response", "task_planner_instance"]:
                    continue
                # Name similarity
                if any(word in name.lower() for word in query_lower.split()):
                    suggestions.append(name)
                    continue

                # Description similarity
                if var_def and any(word in str(var_def).lower() for word in query_lower.split()):
                    suggestions.append(name)
                    continue


        return list(set(suggestions))[:10]

    def _document_structure(self, data: Any, path_prefix: str, docs: dict[str, dict]):
        """A recursive helper to document nested dictionaries and lists."""
        if isinstance(data, dict):
            for key, value in data.items():
                # Construct the full path for the current item
                current_path = f"{path_prefix}.{key}" if path_prefix else key

                # Generate a preview for the value
                if isinstance(value, str):
                    preview = value[:70] + "..." if len(value) > 70 else value
                elif isinstance(value, dict):
                    preview = f"Object with keys: {list(value.keys())[:3]}" + ("..." if len(value.keys()) > 3 else "")
                elif isinstance(value, list):
                    preview = f"List with {len(value)} items"
                else:
                    preview = str(value)

                # Store the documentation for the current path
                docs[current_path] = {
                    'preview': preview,
                    'type': type(value).__name__
                }

                # Recurse into nested structures
                if isinstance(value, dict | list):
                    self._document_structure(value, current_path, docs)

        elif isinstance(data, list):
            for i, item in enumerate(data):
                # Construct the full path for the list item
                current_path = f"{path_prefix}.{i}"

                # Generate a preview for the item
                if isinstance(item, str):
                    preview = item[:70] + "..." if len(item) > 70 else item
                elif isinstance(item, dict):
                    preview = f"Object with keys: {list(item.keys())[:3]}" + ("..." if len(item.keys()) > 3 else "")
                elif isinstance(item, list):
                    preview = f"List with {len(item)} items"
                else:
                    preview = str(item)

                docs[current_path] = {
                    'preview': preview,
                    'type': type(item).__name__
                }

                # Recurse into nested structures
                if isinstance(item, dict | list):
                    self._document_structure(item, current_path, docs)

    def get_available_variables(self) -> dict[str, dict]:
        """
        Recursively documents all available variables from world_model and scopes
        to provide a comprehensive overview for an LLM.
        """
        all_vars_docs = {}

        # 1. Document the world_model (top-level variables)
        # self._document_structure(self.world_model, "", all_vars_docs)

        # 2. Document each scope
        for scope_name, scope_data in self.scopes.items():
            # Add documentation for the scope root itself
            if scope_name == "shared":
                continue
            if isinstance(scope_data, dict):
                pass
            elif isinstance(scope_data, list):
                pass
            elif isinstance(scope_data, str | int):
                pass
            else:
                continue

            all_vars_docs[scope_name] = scope_data

            # Recurse into the scope's data
            # self._document_structure(scope_data, scope_name, all_vars_docs)

        return all_vars_docs

    def get_llm_variable_context(self) -> str:
        """
        Generates a detailed variable context formatted for LLM consumption,
        explaining structure, access patterns, and listing all available variables.
        """
        context_parts = [
            "## Variable System Reference",
            "You can access a state management system to retrieve data using dot notation.",
            "Syntax: `{{ path.to.variable }}` or `$path.to.variable`.",
            "",
            "### How to Access Data",
            "The system contains nested objects (dictionaries) and lists (arrays).",
            "",
            "**1. Object (Dictionary) Access (Primary Usage):**",
            "Use a dot (`.`) to access values inside an object. This is the most common way to get data.",
            "Example: If a `user` object exists with a `profile`, you can get the name with `{{ user.profile.name }}`.",
            "",
            "**2. List (Array) Access:**",
            "If a variable is a list, use a dot (`.`) followed by a zero-based number (index) to access a specific item.",
            "Example: To get the first email from a user's email list, use `{{ user.emails.0 }}`.",
            "You can chain these access methods: `{{ user.emails.0.address }}`.",
            "",
            "### Available Variables",
            "Below is a list of all currently available variable paths, their type, and a preview of their content. (Note: Previews may be truncated).",
        ]

        variables = self.get_available_variables()
        if not variables:
            context_parts.append("- No variables are currently set.")
            return "\n".join(context_parts)

        if "shared" in variables:
            variables["shared"] = {'preview': "Shared state variables", 'type': "dict"}

        # yaml dump preview
        context_parts.append("```yaml")
        context_parts.append(yaml.dump(variables, default_flow_style=False, sort_keys=False))
        context_parts.append("```")

        # Add any final complex examples or notes
        context_parts.extend([
            "",
            "**Note on Task Results:**",
            "All task results are stored in the `results` scope. To access the data from a task, append `.data`.",
            "Example: `{{ results.'task-id-123'.data }}`"
        ])

        return "\n".join(context_parts)

format_text(text, context=None)

Enhanced text formatting with multiple syntaxes

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

def format_text(self, text: str, context: dict = None) -> str:
    """Enhanced text formatting with multiple syntaxes"""
    if not text or not isinstance(text, str):
        return str(text) if text is not None else ""

    # Temporary context overlay
    if context:
        original_scopes = self.scopes.copy()
        self.scopes['context'] = context

    try:
        # Handle {{ variable }} syntax
        formatted = self._format_double_braces(text)

        # Handle {variable} syntax
        formatted = self._format_single_braces(formatted)

        # Handle $variable syntax
        formatted = self._format_dollar_syntax(formatted)

        return formatted

    finally:
        if context:
            self.scopes = original_scopes

get(path, default=None, use_cache=True)

Get variable with dot notation path support for dicts and lists.

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

def get(self, path: str, default=None, use_cache: bool = True):
    """Get variable with dot notation path support for dicts and lists."""
    if use_cache and path in self._cache:
        return self._cache[path]

    try:
        value = self._resolve_path(path)
        if use_cache:
            self._cache[path] = value
        return value
    except (KeyError, IndexError):
        # A KeyError or IndexError during resolution means the path is invalid
        return default

get_available_variables()

Recursively documents all available variables from world_model and scopes to provide a comprehensive overview for an LLM.

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

def get_available_variables(self) -> dict[str, dict]:
    """
    Recursively documents all available variables from world_model and scopes
    to provide a comprehensive overview for an LLM.
    """
    all_vars_docs = {}

    # 1. Document the world_model (top-level variables)
    # self._document_structure(self.world_model, "", all_vars_docs)

    # 2. Document each scope
    for scope_name, scope_data in self.scopes.items():
        # Add documentation for the scope root itself
        if scope_name == "shared":
            continue
        if isinstance(scope_data, dict):
            pass
        elif isinstance(scope_data, list):
            pass
        elif isinstance(scope_data, str | int):
            pass
        else:
            continue

        all_vars_docs[scope_name] = scope_data

        # Recurse into the scope's data
        # self._document_structure(scope_data, scope_name, all_vars_docs)

    return all_vars_docs

get_llm_variable_context()

Generates a detailed variable context formatted for LLM consumption, explaining structure, access patterns, and listing all available variables.

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

def get_llm_variable_context(self) -> str:
    """
    Generates a detailed variable context formatted for LLM consumption,
    explaining structure, access patterns, and listing all available variables.
    """
    context_parts = [
        "## Variable System Reference",
        "You can access a state management system to retrieve data using dot notation.",
        "Syntax: `{{ path.to.variable }}` or `$path.to.variable`.",
        "",
        "### How to Access Data",
        "The system contains nested objects (dictionaries) and lists (arrays).",
        "",
        "**1. Object (Dictionary) Access (Primary Usage):**",
        "Use a dot (`.`) to access values inside an object. This is the most common way to get data.",
        "Example: If a `user` object exists with a `profile`, you can get the name with `{{ user.profile.name }}`.",
        "",
        "**2. List (Array) Access:**",
        "If a variable is a list, use a dot (`.`) followed by a zero-based number (index) to access a specific item.",
        "Example: To get the first email from a user's email list, use `{{ user.emails.0 }}`.",
        "You can chain these access methods: `{{ user.emails.0.address }}`.",
        "",
        "### Available Variables",
        "Below is a list of all currently available variable paths, their type, and a preview of their content. (Note: Previews may be truncated).",
    ]

    variables = self.get_available_variables()
    if not variables:
        context_parts.append("- No variables are currently set.")
        return "\n".join(context_parts)

    if "shared" in variables:
        variables["shared"] = {'preview': "Shared state variables", 'type': "dict"}

    # yaml dump preview
    context_parts.append("```yaml")
    context_parts.append(yaml.dump(variables, default_flow_style=False, sort_keys=False))
    context_parts.append("```")

    # Add any final complex examples or notes
    context_parts.extend([
        "",
        "**Note on Task Results:**",
        "All task results are stored in the `results` scope. To access the data from a task, append `.data`.",
        "Example: `{{ results.'task-id-123'.data }}`"
    ])

    return "\n".join(context_parts)

get_scope_info()

Get information about all available scopes

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

def get_scope_info(self) -> dict[str, Any]:
    """Get information about all available scopes"""
    info = {}
    for scope_name, scope_data in self.scopes.items():
        if isinstance(scope_data, dict):
            info[scope_name] = {
                'type': 'dict',
                'keys': len(scope_data),
                'sample_keys': list(scope_data.keys())[:5]
            }
        else:
            info[scope_name] = {
                'type': type(scope_data).__name__,
                'value': str(scope_data)[:100]
            }
    return info

get_variable_suggestions(query)

Get variable suggestions based on query content

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

def get_variable_suggestions(self, query: str) -> list[str]:
    """Get variable suggestions based on query content"""

    query_lower = query.lower()
    suggestions = []

    # Check all variables for relevance
    for scope in self.scopes.values():
        for name, var_def in scope.items():
            if name in ["system_context", "task_executor_instance",
                        "index", "tool_capabilities", "use_fast_response", "task_planner_instance"]:
                continue
            # Name similarity
            if any(word in name.lower() for word in query_lower.split()):
                suggestions.append(name)
                continue

            # Description similarity
            if var_def and any(word in str(var_def).lower() for word in query_lower.split()):
                suggestions.append(name)
                continue


    return list(set(suggestions))[:10]

register_scope(name, data)

Register a new variable scope

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

def register_scope(self, name: str, data: dict):
    """Register a new variable scope"""
    self.scopes[name] = data
    self._cache.clear()

set(path, value, create_scope=True)

Set variable with dot notation path support for dicts and lists.

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

def set(self, path: str, value, create_scope: bool = True):
    """Set variable with dot notation path support for dicts and lists."""
    # Invalidate cache for this path
    if path in self._cache:
        del self._cache[path]

    parts = path.split('.')

    if len(parts) == 1:
        # Simple key in world_model
        self.world_model[path] = value
        return

    scope_name = parts[0]
    if scope_name not in self.scopes:
        if create_scope:
            self.scopes[scope_name] = {}
        else:
            raise KeyError(f"Scope '{scope_name}' not found")

    current = self.scopes[scope_name]

    # Iterate to the second-to-last part to get the container
    for i, part in enumerate(parts[1:-1]):
        next_part = parts[i + 2]  # Look ahead to the next part in the path

        # Determine if the current part is a dictionary key or a list index
        try:
            # Try to treat it as a list index
            key = int(part)
            if not isinstance(current, list):
                # If current is not a list, we can't use an integer index
                raise TypeError(f"Attempted to use integer index '{key}' on non-list for path '{path}'")

            # Ensure list is long enough
            while len(current) <= key:
                current.append(None)  # Pad with None

            # If the next level doesn't exist, create it based on the next part
            if current[key] is None:
                current[key] = [] if next_part.isdigit() else {}

            current = current[key]

        except ValueError:
            # It's a dictionary key
            key = part
            if not isinstance(current, dict):
                raise TypeError(f"Attempted to use string key '{key}' on non-dict for path '{path}'")

            if key not in current:
                # Create the next level: a list if the next part is a number, else a dict
                current[key] = [] if next_part.isdigit() else {}

            current = current[key]

    # Handle the final part (the actual assignment)
    last_part = parts[-1]

    if isinstance(current, list):
        try:
            key = int(last_part)
            while len(current) <= key:
                current.append(None)
            current[key] = value
        except ValueError:
            current.append(value)
    elif isinstance(current, dict):
        current[last_part] = value
    elif scope_name == 'tasks' and hasattr(current, 'task_identification_attr'):# from tasks like Tooltask ... model dump and acces
        dict_data = asdict(current)
        dict_data[last_part] = value
        current = dict_data
        # update self.scopes['tasks'] with the updated task
        self.scopes['tasks'][parts[1]][last_part] = current
    else:
        raise TypeError(f"Final container is not a list or dictionary for path '{path}' its a {type(current)}")

    self._cache.clear()

set_results_store(results_store)

Set the results store for task result references

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

def set_results_store(self, results_store: dict):
    """Set the results store for task result references"""
    self.scopes['results'] = results_store
    self._cache.clear()

set_tasks_store(tasks_store)

Set tasks store for task metadata access

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

def set_tasks_store(self, tasks_store: dict):
    """Set tasks store for task metadata access"""
    self.scopes['tasks'] = tasks_store
    self._cache.clear()

validate_references(text)

Validate all variable references in text

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

def validate_references(self, text: str) -> dict[str, bool]:
    """Validate all variable references in text"""
    import re

    references = {}

    # Find all {{ }} references
    double_brace_refs = re.findall(r'\{\{\s*([^}]+)\s*\}\}', text)
    for ref in double_brace_refs:
        references["{{"+ref+"}}"] = self.get(ref.strip()) is not None

    # Find all {} references
    single_brace_refs = re.findall(r'\{([^{}\s]+)\}', text)
    for ref in single_brace_refs:
        if '.' not in ref:  # Only simple vars
            references["{"+ref+"}"] = self.get(ref.strip()) is not None

    # Find all $ references
    dollar_refs = re.findall(r'\$([a-zA-Z_][a-zA-Z0-9_]*)', text)
    for ref in dollar_refs:
        references[f"${ref}"] = self.get(ref) is not None

    return references

auto_unescape(args)

Automatically unescape all strings in nested data structure.

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

def auto_unescape(args: Any) -> Any:
    """Automatically unescape all strings in nested data structure."""
    return process_nested(args)

create_task(task_type, **kwargs)

Factory für Task-Erstellung mit korrektem Typ

Source code in toolboxv2/mods/isaa/base/Agent/types.py

def create_task(task_type: str, **kwargs) -> Task:
    """Factory für Task-Erstellung mit korrektem Typ"""
    task_classes = {
        "llm_call": LLMTask,
        "tool_call": ToolTask,
        "decision": DecisionTask,
        "generic": Task,
        "LLMTask": LLMTask,
        "ToolTask": ToolTask,
        "DecisionTask": DecisionTask,
        "Task": Task,
    }

    task_class = task_classes.get(task_type, Task)

    # Standard-Felder setzen
    if "id" not in kwargs:
        kwargs["id"] = str(uuid.uuid4())
    if "type" not in kwargs:
        kwargs["type"] = task_type
    if "critical" not in kwargs:
        kwargs["critical"] = task_type in ["llm_call", "decision"]

    # Ensure metadata is initialized
    if "metadata" not in kwargs:
        kwargs["metadata"] = {}

    # Create task and ensure post_init is called
    task = task_class(**kwargs)

    # Double-check metadata initialization
    if not hasattr(task, 'metadata') or task.metadata is None:
        task.metadata = {}

    return task

get_args_schema(func)

Generate a string representation of a function's arguments and annotations. Keeps args and *kwargs indicators and handles modern Python type hints.

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

def get_args_schema(func: Callable) -> str:
    """
    Generate a string representation of a function's arguments and annotations.
    Keeps *args and **kwargs indicators and handles modern Python type hints.
    """
    sig = inspect.signature(func)
    parts = []

    for name, param in sig.parameters.items():
        ann = ""
        if param.annotation is not inspect._empty:
            ann = f": {_annotation_to_str(param.annotation)}"

        default = ""
        if param.default is not inspect._empty:
            default = f" = {repr(param.default)}"

        prefix = ""
        if param.kind == inspect.Parameter.VAR_POSITIONAL:
            prefix = "*"
        elif param.kind == inspect.Parameter.VAR_KEYWORD:
            prefix = "**"

        parts.append(f"{prefix}{name}{ann}{default}")

    return f"({', '.join(parts)})"

get_progress_summary(self)

Get comprehensive progress summary from the agent

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

def get_progress_summary(self) -> dict[str, Any]:
    """Get comprehensive progress summary from the agent"""
    if hasattr(self, 'progress_tracker'):
        return self.progress_tracker.get_summary()
    return {"error": "No progress tracker available"}

needs_unescaping(text)

Detect if string likely needs unescaping.

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

def needs_unescaping(text: str) -> bool:
    """Detect if string likely needs unescaping."""
    return bool(re.search(r'\\[ntr"\'\\]', text)) or len(text) > 50

process_nested(data, max_depth=20)

Recursively process nested structures, unescaping strings that need it.

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

def process_nested(data: Any, max_depth: int = 20) -> Any:
    """Recursively process nested structures, unescaping strings that need it."""
    if max_depth <= 0:
        return data

    if isinstance(data, dict):
        return {k: process_nested(v, max_depth - 1) for k, v in data.items()}

    elif isinstance(data, list | tuple):
        processed = [process_nested(item, max_depth - 1) for item in data]
        return type(data)(processed)

    elif isinstance(data, str) and needs_unescaping(data):
        return unescape_string(data)

    return data

unescape_string(text)

Universal string unescaping for any programming language.

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

def unescape_string(text: str) -> str:
    """Universal string unescaping for any programming language."""
    if not isinstance(text, str) or len(text) < 2:
        return text

    # Remove outer quotes if wrapped
    if (text.startswith('"') and text.endswith('"')) or (text.startswith("'") and text.endswith("'")):
        text = text[1:-1]

    # Universal escape sequences
    escapes = {
        '\\n': '\n', '\\t': '\t', '\\r': '\r',
        '\\"': '"', "\\'": "'", '\\\\': '\\'
    }

    for escaped, unescaped in escapes.items():
        text = text.replace(escaped, unescaped)

    return text

with_progress_tracking(cls)

Ein Klassendekorator, der die Methoden run_async, prep_async, exec_async, und exec_fallback_async automatisch mit umfassendem Progress-Tracking umwickelt.

Source code in toolboxv2/mods/isaa/base/Agent/agent.py

def with_progress_tracking(cls):
    """
    Ein Klassendekorator, der die Methoden run_async, prep_async, exec_async,
    und exec_fallback_async automatisch mit umfassendem Progress-Tracking umwickelt.
    """

    # --- Wrapper für run_async ---
    original_run = getattr(cls, 'run_async', None)
    if original_run:
        @functools.wraps(original_run)
        async def wrapped_run_async(self, shared):
            progress_tracker = shared.get("progress_tracker")
            node_name = self.__class__.__name__

            if not progress_tracker:
                return await original_run(self, shared)

            timer_key = f"{node_name}_total"
            progress_tracker.start_timer(timer_key)
            await progress_tracker.emit_event(ProgressEvent(
                event_type="node_enter",
                timestamp=time.time(),
                node_name=node_name,
                session_id=shared.get("session_id"),
                task_id=shared.get("current_task_id"),
                plan_id=shared.get("current_plan", TaskPlan(id="none", name="none", description="none")).id if shared.get("current_plan") else None,
                status=NodeStatus.RUNNING,
                success=None
            ))

            try:
                # Hier wird die ursprüngliche Methode aufgerufen
                result = await original_run(self, shared)

                total_duration = progress_tracker.end_timer(timer_key)
                await progress_tracker.emit_event(ProgressEvent(
                    event_type="node_exit",
                    timestamp=time.time(),
                    node_name=node_name,
                    status=NodeStatus.COMPLETED,
                    success=True,
                    node_duration=total_duration,
                    routing_decision=result,
                    session_id=shared.get("session_id"),
                    task_id=shared.get("current_task_id"),
                    metadata={"success": True}
                ))

                return result
            except Exception as e:
                total_duration = progress_tracker.end_timer(timer_key)
                await progress_tracker.emit_event(ProgressEvent(
                    event_type="error",
                    timestamp=time.time(),
                    node_name=node_name,
                    status=NodeStatus.FAILED,
                    success=False,
                    node_duration=total_duration,
                    session_id=shared.get("session_id"),
                    metadata={"error": str(e), "error_type": type(e).__name__}
                ))
                raise

        cls.run_async = wrapped_run_async

    # --- Wrapper für prep_async ---
    original_prep = getattr(cls, 'prep_async', None)
    if original_prep:
        @functools.wraps(original_prep)
        async def wrapped_prep_async(self, shared):
            progress_tracker = shared.get("progress_tracker")
            node_name = self.__class__.__name__

            if not progress_tracker:
                return await original_prep(self, shared)
            timer_key = f"{node_name}_total_p"
            progress_tracker.start_timer(timer_key)
            timer_key = f"{node_name}_prep"
            progress_tracker.start_timer(timer_key)
            await progress_tracker.emit_event(ProgressEvent(
                event_type="node_phase",
                timestamp=time.time(),
                node_name=node_name,
                status=NodeStatus.STARTING,
                node_phase="prep",
                session_id=shared.get("session_id")
            ))

            try:
                result = await original_prep(self, shared)

                prep_duration = progress_tracker.end_timer(timer_key)
                await progress_tracker.emit_event(ProgressEvent(
                    event_type="node_phase",
                    timestamp=time.time(),
                    status=NodeStatus.RUNNING,
                    success=True,
                    node_name=node_name,
                    node_phase="prep_complete",
                    node_duration=prep_duration,
                    session_id=shared.get("session_id")
                ))
                return result
            except Exception as e:
                progress_tracker.end_timer(timer_key)
                await progress_tracker.emit_event(ProgressEvent(
                    event_type="error",
                    timestamp=time.time(),
                    node_name=node_name,
                    status=NodeStatus.FAILED,
                    success=False,
                    metadata={"error": str(e), "error_type": type(e).__name__},
                    node_phase="prep_failed"
                ))
                raise


        cls.prep_async = wrapped_prep_async

    # --- Wrapper für exec_async ---
    original_exec = getattr(cls, 'exec_async', None)
    if original_exec:
        @functools.wraps(original_exec)
        async def wrapped_exec_async(self, prep_res):
            progress_tracker = prep_res.get("progress_tracker") if isinstance(prep_res, dict) else None
            node_name = self.__class__.__name__

            if not progress_tracker:
                return await original_exec(self, prep_res)

            timer_key = f"{node_name}_exec"
            progress_tracker.start_timer(timer_key)
            await progress_tracker.emit_event(ProgressEvent(
                event_type="node_phase",
                timestamp=time.time(),
                node_name=node_name,
                status=NodeStatus.RUNNING,
                node_phase="exec",
                session_id=prep_res.get("session_id") if isinstance(prep_res, dict) else None
            ))

            # In exec gibt es normalerweise keine Fehlerbehandlung, da diese von run_async übernommen wird
            result = await original_exec(self, prep_res)

            exec_duration = progress_tracker.end_timer(timer_key)
            await progress_tracker.emit_event(ProgressEvent(
                event_type="node_phase",
                timestamp=time.time(),
                node_name=node_name,
                status=NodeStatus.RUNNING,
                success=True,
                node_phase="exec_complete",
                node_duration=exec_duration,
                session_id=prep_res.get("session_id") if isinstance(prep_res, dict) else None
            ))
            return result

        cls.exec_async = wrapped_exec_async

    # --- Wrapper für post_async ---
    original_post = getattr(cls, 'post_async', None)
    if original_post:
        @functools.wraps(original_post)
        async def wrapped_post_async(self, shared, prep_res, exec_res):
            if isinstance(exec_res, str):
                print("exec_res is string:", exec_res)
            progress_tracker = shared.get("progress_tracker")
            node_name = self.__class__.__name__

            if not progress_tracker:
                return await original_post(self, shared, prep_res, exec_res)

            timer_key_post = f"{node_name}_post"
            progress_tracker.start_timer(timer_key_post)
            await progress_tracker.emit_event(ProgressEvent(
                event_type="node_phase",
                timestamp=time.time(),
                node_name=node_name,
                status=NodeStatus.COMPLETING,  # Neue Phase "completing"
                node_phase="post",
                session_id=shared.get("session_id")
            ))

            try:
                # Die eigentliche post_async Methode aufrufen
                result = await original_post(self, shared, prep_res, exec_res)

                post_duration = progress_tracker.end_timer(timer_key_post)
                total_duration = progress_tracker.end_timer(f"{node_name}_total_p")  # Gesamtdauer stoppen

                # Sende das entscheidende "node_exit" Event nach erfolgreicher post-Phase
                await progress_tracker.emit_event(ProgressEvent(
                    event_type="node_exit",
                    timestamp=time.time(),
                    node_name=node_name,
                    status=NodeStatus.COMPLETED,
                    success=True,
                    node_duration=total_duration,
                    routing_decision=result,
                    session_id=shared.get("session_id"),
                    task_id=shared.get("current_task_id"),
                    metadata={
                        "success": True,
                        "post_duration": post_duration
                    }
                ))

                return result
            except Exception as e:
                # Fehler in der post-Phase

                post_duration = progress_tracker.end_timer(timer_key_post)
                total_duration = progress_tracker.end_timer(f"{node_name}_total")
                await progress_tracker.emit_event(ProgressEvent(
                    event_type="error",
                    timestamp=time.time(),
                    node_name=node_name,
                    status=NodeStatus.FAILED,
                    success=False,
                    node_duration=total_duration,
                    metadata={"error": str(e), "error_type": type(e).__name__, "phase": "post"},
                    node_phase="post_failed"
                ))
                raise

        cls.post_async = wrapped_post_async

    # --- Wrapper für exec_fallback_async ---
    original_fallback = getattr(cls, 'exec_fallback_async', None)
    if original_fallback:
        @functools.wraps(original_fallback)
        async def wrapped_fallback_async(self, prep_res, exc):
            progress_tracker = prep_res.get("progress_tracker") if isinstance(prep_res, dict) else None
            node_name = self.__class__.__name__

            if progress_tracker:
                timer_key = f"{node_name}_exec"
                exec_duration = progress_tracker.end_timer(timer_key)
                await progress_tracker.emit_event(ProgressEvent(
                    event_type="node_phase",
                    timestamp=time.time(),
                    node_name=node_name,
                    node_phase="exec_fallback",
                    node_duration=exec_duration,
                    status=NodeStatus.FAILED,
                    success=False,
                    session_id=prep_res.get("session_id") if isinstance(prep_res, dict) else None,
                    metadata={"error": str(exc), "error_type": type(exc).__name__},
                ))

            return await original_fallback(self, prep_res, exc)

        cls.exec_fallback_async = wrapped_fallback_async

    return cls

builder

A2AConfig

Bases: BaseModel

A2A server configuration

Source code in toolboxv2/mods/isaa/base/Agent/builder.py

class A2AConfig(BaseModel):
    """A2A server configuration"""
    model_config = ConfigDict(arbitrary_types_allowed=True)

    enabled: bool = False
    host: str = "0.0.0.0"
    port: int = 5000
    agent_name: Optional[str] = None
    agent_description: Optional[str] = None
    agent_version: str = "1.0.0"
    expose_tools_as_skills: bool = True

AgentConfig

Bases: BaseModel

Complete agent configuration for loading/saving

Source code in toolboxv2/mods/isaa/base/Agent/builder.py

class AgentConfig(BaseModel):
    """Complete agent configuration for loading/saving"""
    model_config = ConfigDict(arbitrary_types_allowed=True)

    # Basic settings
    name: str = "ProductionAgent"
    description: str = "Production-ready PocketFlow agent"
    version: str = "2.0.0"

    # LLM settings
    fast_llm_model: str = "openrouter/anthropic/claude-3-haiku"
    complex_llm_model: str = "openrouter/openai/gpt-4o"
    system_message: str = """You are a production-ready autonomous agent with advanced capabilities including:
- Native MCP tool integration for extensible functionality
- A2A compatibility for agent-to-agent communication
- Dynamic task planning and execution with adaptive reflection
- Advanced context management with session awareness
- Variable system for dynamic content generation
- Checkpoint/resume capabilities for reliability

Always utilize available tools when they can help solve the user's request efficiently."""

    temperature: float = 0.7
    max_tokens_output: int = 2048
    max_tokens_input: int = 32768
    api_key_env_var: str | None = "OPENROUTER_API_KEY"
    use_fast_response: bool = True

    # Features
    mcp: MCPConfig = Field(default_factory=MCPConfig)
    a2a: A2AConfig = Field(default_factory=A2AConfig)
    telemetry: TelemetryConfig = Field(default_factory=TelemetryConfig)
    checkpoint: CheckpointConfig = Field(default_factory=CheckpointConfig)

    # Agent behavior
    max_parallel_tasks: int = 3
    verbose_logging: bool = False

    # Persona and formatting
    active_persona: Optional[str] = None
    persona_profiles: dict[str, dict[str, Any]] = Field(default_factory=dict)
    default_format_config: Optional[dict[str, Any]] = None

    # Custom variables and world model
    custom_variables: dict[str, Any] = Field(default_factory=dict)
    initial_world_model: dict[str, Any] = Field(default_factory=dict)

CheckpointConfig

Bases: BaseModel

Checkpoint configuration

Source code in toolboxv2/mods/isaa/base/Agent/builder.py

class CheckpointConfig(BaseModel):
    """Checkpoint configuration"""
    enabled: bool = True
    interval_seconds: int = 300  # 5 minutes
    max_checkpoints: int = 10
    checkpoint_dir: str = "./checkpoints"
    auto_save_on_exit: bool = True

FlowAgentBuilder

Production-ready FlowAgent builder focused on MCP, A2A, and robust deployment

Source code in toolboxv2/mods/isaa/base/Agent/builder.py

class FlowAgentBuilder:
    """Production-ready FlowAgent builder focused on MCP, A2A, and robust deployment"""

    def __init__(self, config: AgentConfig = None, config_path: str = None):
        """Initialize builder with configuration"""

        if config and config_path:
            raise ValueError("Provide either config object or config_path, not both")

        if config_path:
            self.config = self.load_config(config_path)
        elif config:
            self.config = config
        else:
            self.config = AgentConfig()

        # Runtime components
        self._custom_tools: dict[str, tuple[Callable, str]] = {}
        self._mcp_tools: dict[str, dict] = {}
        from toolboxv2.mods.isaa.extras.mcp_session_manager import MCPSessionManager

        self._mcp_session_manager = MCPSessionManager()

        self._budget_manager: BudgetManager = None
        self._tracer_provider: TracerProvider = None
        self._a2a_server: Any = None

        # Set logging level
        if self.config.verbose_logging:
            logging.getLogger().setLevel(logging.DEBUG)

        iprint(f"FlowAgent Builder initialized: {self.config.name}")

    # ===== CONFIGURATION MANAGEMENT =====

    def load_config(self, config_path: str) -> AgentConfig:
        """Load agent configuration from file"""
        path = Path(config_path)
        if not path.exists():
            raise FileNotFoundError(f"Config file not found: {config_path}")

        try:
            with open(path, encoding='utf-8') as f:
                if path.suffix.lower() in ['.yaml', '.yml']:
                    data = yaml.safe_load(f)
                else:
                    data = json.load(f)

            return AgentConfig(**data)

        except Exception as e:
            eprint(f"Failed to load config from {config_path}: {e}")
            raise

    def save_config(self, config_path: str, format: str = 'yaml'):
        """Save current configuration to file"""
        path = Path(config_path)
        path.parent.mkdir(parents=True, exist_ok=True)

        try:
            data = self.config.model_dump()

            with open(path, 'w', encoding='utf-8') as f:
                if format.lower() == 'yaml':
                    yaml.dump(data, f, default_flow_style=False, indent=2)
                else:
                    json.dump(data, f, indent=2)

            iprint(f"Configuration saved to {config_path}")

        except Exception as e:
            eprint(f"Failed to save config to {config_path}: {e}")
            raise

    @classmethod
    def from_config_file(cls, config_path: str) -> 'FlowAgentBuilder':
        """Create builder from configuration file"""
        return cls(config_path=config_path)

    # ===== FLUENT BUILDER API =====

    def with_name(self, name: str) -> 'FlowAgentBuilder':
        """Set agent name"""
        self.config.name = name
        return self

    def with_models(self, fast_model: str, complex_model: str = None) -> 'FlowAgentBuilder':
        """Set LLM models"""
        self.config.fast_llm_model = fast_model
        if complex_model:
            self.config.complex_llm_model = complex_model
        return self

    def with_system_message(self, message: str) -> 'FlowAgentBuilder':
        """Set system message"""
        self.config.system_message = message
        return self

    def with_temperature(self, temp: float) -> 'FlowAgentBuilder':
        """Set temperature"""
        self.config.temperature = temp
        return self

    def with_budget_manager(self, max_cost: float = 10.0) -> 'FlowAgentBuilder':
        """Enable budget management"""
        if LITELLM_AVAILABLE:
            self._budget_manager = BudgetManager("agent")
            iprint(f"Budget manager enabled: ${max_cost}")
        else:
            wprint("LiteLLM not available, budget manager disabled")
        return self

    def verbose(self, enable: bool = True) -> 'FlowAgentBuilder':
        """Enable verbose logging"""
        self.config.verbose_logging = enable
        if enable:
            logging.getLogger().setLevel(logging.DEBUG)
        return self

    # ===== MCP INTEGRATION =====

    def enable_mcp_server(self, host: str = "0.0.0.0", port: int = 8000,
                          server_name: str = None) -> 'FlowAgentBuilder':
        """Enable MCP server"""
        if not MCP_AVAILABLE:
            wprint("MCP not available, cannot enable server")
            return self

        self.config.mcp.enabled = True
        self.config.mcp.host = host
        self.config.mcp.port = port
        self.config.mcp.server_name = server_name or f"{self.config.name}_MCP"

        iprint(f"MCP server enabled: {host}:{port}")
        return self

    async def _load_mcp_server_capabilities(self, server_name: str, server_config: dict[str, Any]):
        """Load all capabilities from MCP server with persistent session"""
        try:
            # Get or create persistent session
            session = await self._mcp_session_manager.get_session(server_name, server_config)
            if not session:
                eprint(f"Failed to create session for MCP server: {server_name}")
                return

            # Extract all capabilities
            capabilities = await self._mcp_session_manager.extract_capabilities(session, server_name)

            # Create tool wrappers
            for tool_name, tool_info in capabilities['tools'].items():
                wrapper_name = f"{server_name}_{tool_name}"
                tool_wrapper = self._create_tool_wrapper(server_name, tool_name, tool_info, session)
                self._mcp_tools[wrapper_name] = {
                    'function': tool_wrapper,
                    'description': tool_info['description'],
                    'type': 'tool',
                    'server': server_name,
                    'original_name': tool_name,
                    'input_schema': tool_info.get('input_schema'),
                    'output_schema': tool_info.get('output_schema')
                }

            # Create resource wrappers
            for resource_uri, resource_info in capabilities['resources'].items():
                wrapper_name = f"{server_name}_resource_{resource_info['name'].replace('/', '_')}"
                resource_wrapper = self._create_resource_wrapper(server_name, resource_uri, resource_info, session)

                self._mcp_tools[wrapper_name] = {
                    'function': resource_wrapper,
                    'description': f"Read resource: {resource_info['description']}",
                    'type': 'resource',
                    'server': server_name,
                    'original_uri': resource_uri
                }

            # Create resource template wrappers
            for template_uri, template_info in capabilities['resource_templates'].items():
                wrapper_name = f"{server_name}_template_{template_info['name'].replace('/', '_')}"
                template_wrapper = self._create_resource_template_wrapper(server_name, template_uri, template_info,
                                                                          session)

                self._mcp_tools[wrapper_name] = {
                    'function': template_wrapper,
                    'description': f"Access resource template: {template_info['description']}",
                    'type': 'resource_template',
                    'server': server_name,
                    'original_template': template_uri
                }

            # Create prompt wrappers
            for prompt_name, prompt_info in capabilities['prompts'].items():
                wrapper_name = f"{server_name}_prompt_{prompt_name}"
                prompt_wrapper = self._create_prompt_wrapper(server_name, prompt_name, prompt_info, session)

                self._mcp_tools[wrapper_name] = {
                    'function': prompt_wrapper,
                    'description': f"Execute prompt: {prompt_info['description']}",
                    'type': 'prompt',
                    'server': server_name,
                    'original_name': prompt_name,
                    'arguments': prompt_info.get('arguments', [])
                }

            total_capabilities = (len(capabilities['tools']) +
                                  len(capabilities['resources']) +
                                  len(capabilities['resource_templates']) +
                                  len(capabilities['prompts']))

            iprint(f"Created {total_capabilities} capability wrappers for server: {server_name}")

        except Exception as e:
            eprint(f"Failed to load capabilities from MCP server {server_name}: {e}")

    def _create_tool_wrapper(self, server_name: str, tool_name: str, tool_info: dict, session: ClientSession):
        """Create wrapper function for MCP tool with dynamic signature based on schema"""
        import inspect

        # Extract parameter information from input schema
        input_schema = tool_info.get('input_schema', {})
        output_schema = tool_info.get('output_schema', {})

        # Build parameter list
        parameters = []
        required_params = set(input_schema.get('required', []))
        properties = input_schema.get('properties', {})

        # Create parameters with proper types
        for param_name, param_info in properties.items():
            param_type = param_info.get('type', 'string')
            python_type = {
                'string': str,
                'integer': int,
                'number': float,
                'boolean': bool,
                'array': list,
                'object': dict
            }.get(param_type, str)

            # Determine if parameter is required
            if param_name in required_params:
                param = inspect.Parameter(param_name, inspect.Parameter.POSITIONAL_OR_KEYWORD, annotation=python_type)
            else:
                # Optional parameters get default None
                param = inspect.Parameter(param_name, inspect.Parameter.POSITIONAL_OR_KEYWORD,
                                          annotation=python_type, default=None)
            parameters.append(param)

        # Determine return type from output schema
        return_type = str  # Default
        if output_schema and 'properties' in output_schema:
            output_props = output_schema['properties']
            if len(output_props) == 1:
                # Single property, return its type directly
                prop_info = list(output_props.values())[0]
                prop_type = prop_info.get('type', 'string')
                return_type = {
                    'string': str,
                    'integer': int,
                    'number': float,
                    'boolean': bool,
                    'array': list,
                    'object': dict
                }.get(prop_type, str)
            else:
                # Multiple properties, return dict
                return_type = dict

        # Create the actual function
        async def tool_wrapper(*args, **kwargs):
            try:
                # Map arguments to schema parameters
                arguments = {}
                param_names = list(properties.keys())

                # Map positional args
                for i, arg in enumerate(args):
                    if i < len(param_names):
                        arguments[param_names[i]] = arg

                # Add keyword arguments, filtering out None for optional params
                for key, value in kwargs.items():
                    if value is not None or key in required_params:
                        arguments[key] = value

                # Validate required parameters
                missing_required = required_params - set(arguments.keys())
                if missing_required:
                    raise ValueError(f"Missing required parameters: {missing_required}")

                # Call the actual MCP tool
                result = await session.call_tool(tool_name, arguments)

                # Handle structured vs unstructured results
                if hasattr(result, 'structuredContent') and result.structuredContent:
                    structured_data = result.structuredContent

                    # If output schema expects single property, extract it
                    if output_schema and 'properties' in output_schema:
                        output_props = output_schema['properties']
                        if len(output_props) == 1:
                            prop_name = list(output_props.keys())[0]
                            if isinstance(structured_data, dict) and prop_name in structured_data:
                                return structured_data[prop_name]

                    return structured_data

                # Fallback to content extraction
                if result.content:
                    content = result.content[0]
                    if hasattr(content, 'text'):
                        return content.text
                    elif hasattr(content, 'data'):
                        return content.data
                    else:
                        return str(content)

                return "No content returned"

            except Exception as e:
                eprint(f"MCP tool {server_name}.{tool_name} failed: {e}")
                raise RuntimeError(f"Error executing {tool_name}: {str(e)}")

        # Set dynamic signature
        signature = inspect.Signature(parameters, return_annotation=return_type)
        tool_wrapper.__signature__ = signature
        tool_wrapper.__name__ = f"{server_name}_{tool_name}"
        tool_wrapper.__doc__ = tool_info.get('description', f"MCP tool: {tool_name}")
        tool_wrapper.__annotations__ = {'return': return_type}

        # Add parameter annotations
        for param in parameters:
            tool_wrapper.__annotations__[param.name] = param.annotation

        return tool_wrapper

    def _create_resource_wrapper(self, server_name: str, resource_uri: str, resource_info: dict,
                                 session: ClientSession):
        """Create wrapper function for MCP resource with proper signature"""
        import inspect

        # Resources typically don't take parameters, return string content
        async def resource_wrapper() -> str:
            """Read MCP resource content"""
            try:
                from pydantic import AnyUrl
                result = await session.read_resource(AnyUrl(resource_uri))

                if result.contents:
                    content = result.contents[0]
                    if hasattr(content, 'text'):
                        return content.text
                    elif hasattr(content, 'data'):
                        # Handle binary data
                        if isinstance(content.data, bytes):
                            return content.data.decode('utf-8', errors='ignore')
                        return str(content.data)
                    else:
                        return str(content)

                return ""

            except Exception as e:
                eprint(f"MCP resource {resource_uri} failed: {e}")
                raise RuntimeError(f"Error reading resource: {str(e)}")

        # Set signature and metadata
        signature = inspect.Signature([], return_annotation=str)
        resource_wrapper.__signature__ = signature
        resource_wrapper.__name__ = f"{server_name}_resource_{resource_info['name'].replace('/', '_').replace(':', '_')}"
        resource_wrapper.__doc__ = f"Read MCP resource: {resource_info.get('description', resource_uri)}"
        resource_wrapper.__annotations__ = {'return': str}

        return resource_wrapper

    def _create_resource_template_wrapper(self, server_name: str, template_uri: str, template_info: dict,
                                          session: ClientSession):
        """Create wrapper function for MCP resource template with dynamic parameters"""
        import inspect
        import re

        # Extract template variables from URI (e.g., {owner}, {repo})
        template_vars = re.findall(r'\{(\w+)\}', template_uri)

        # Create parameters for each template variable
        parameters = []
        for var_name in template_vars:
            param = inspect.Parameter(var_name, inspect.Parameter.POSITIONAL_OR_KEYWORD, annotation=str)
            parameters.append(param)

        async def template_wrapper(*args, **kwargs) -> str:
            """Access MCP resource template with parameters"""
            try:
                from pydantic import AnyUrl

                # Map arguments to template variables
                template_args = {}
                for i, arg in enumerate(args):
                    if i < len(template_vars):
                        template_args[template_vars[i]] = arg

                template_args.update(kwargs)

                # Validate all required template variables are provided
                missing_vars = set(template_vars) - set(template_args.keys())
                if missing_vars:
                    raise ValueError(f"Missing required template variables: {missing_vars}")

                # Replace template variables in URI
                actual_uri = template_uri
                for var_name, value in template_args.items():
                    actual_uri = actual_uri.replace(f"{{{var_name}}}", str(value))

                result = await session.read_resource(AnyUrl(actual_uri))

                if result.contents:
                    content = result.contents[0]
                    if hasattr(content, 'text'):
                        return content.text
                    elif hasattr(content, 'data'):
                        if isinstance(content.data, bytes):
                            return content.data.decode('utf-8', errors='ignore')
                        return str(content.data)
                    else:
                        return str(content)

                return ""

            except Exception as e:
                eprint(f"MCP resource template {template_uri} failed: {e}")
                raise RuntimeError(f"Error accessing resource template: {str(e)}")

        # Set dynamic signature
        signature = inspect.Signature(parameters, return_annotation=str)
        template_wrapper.__signature__ = signature
        template_wrapper.__name__ = f"{server_name}_template_{template_info['name'].replace('/', '_').replace(':', '_')}"
        template_wrapper.__doc__ = f"Access MCP resource template: {template_info.get('description', template_uri)}\nTemplate variables: {', '.join(template_vars)}"
        template_wrapper.__annotations__ = {'return': str}

        # Add parameter annotations
        for param in parameters:
            template_wrapper.__annotations__[param.name] = str

        return template_wrapper

    def _create_prompt_wrapper(self, server_name: str, prompt_name: str, prompt_info: dict, session: ClientSession):
        """Create wrapper function for MCP prompt with dynamic parameters"""
        import inspect

        # Extract parameter information from prompt arguments
        prompt_args = prompt_info.get('arguments', [])

        # Create parameters
        parameters = []
        for arg_info in prompt_args:
            arg_name = arg_info['name']
            is_required = arg_info.get('required', False)

            if is_required:
                param = inspect.Parameter(arg_name, inspect.Parameter.POSITIONAL_OR_KEYWORD, annotation=str)
            else:
                param = inspect.Parameter(arg_name, inspect.Parameter.POSITIONAL_OR_KEYWORD,
                                          annotation=str, default=None)
            parameters.append(param)

        async def prompt_wrapper(*args, **kwargs) -> str:
            """Execute MCP prompt with parameters"""
            try:
                # Map arguments
                prompt_arguments = {}
                arg_names = [arg['name'] for arg in prompt_args]

                # Map positional args
                for i, arg in enumerate(args):
                    if i < len(arg_names):
                        prompt_arguments[arg_names[i]] = arg

                # Add keyword arguments, filtering None for optional params
                required_args = {arg['name'] for arg in prompt_args if arg.get('required', False)}
                for key, value in kwargs.items():
                    if value is not None or key in required_args:
                        prompt_arguments[key] = value

                # Validate required parameters
                missing_required = required_args - set(prompt_arguments.keys())
                if missing_required:
                    raise ValueError(f"Missing required prompt arguments: {missing_required}")

                result = await session.get_prompt(prompt_name, prompt_arguments)

                # Extract and combine messages
                messages = []
                for message in result.messages:
                    if hasattr(message.content, 'text'):
                        messages.append(message.content.text)
                    else:
                        messages.append(str(message.content))

                return "\n".join(messages) if messages else ""

            except Exception as e:
                eprint(f"MCP prompt {prompt_name} failed: {e}")
                raise RuntimeError(f"Error executing prompt: {str(e)}")

        # Set dynamic signature
        signature = inspect.Signature(parameters, return_annotation=str)
        prompt_wrapper.__signature__ = signature
        prompt_wrapper.__name__ = f"{server_name}_prompt_{prompt_name}"

        # Build docstring with parameter info
        param_docs = []
        for arg_info in prompt_args:
            required_str = "required" if arg_info.get('required', False) else "optional"
            param_docs.append(
                f"    {arg_info['name']} ({required_str}): {arg_info.get('description', 'No description')}")

        docstring = f"Execute MCP prompt: {prompt_info.get('description', prompt_name)}"
        if param_docs:
            docstring += "\n\nParameters:\n" + "\n".join(param_docs)

        prompt_wrapper.__doc__ = docstring
        prompt_wrapper.__annotations__ = {'return': str}

        # Add parameter annotations
        for param in parameters:
            prompt_wrapper.__annotations__[param.name] = str

        return prompt_wrapper

    def load_mcp_tools_from_config(self, config_path: str | dict) -> 'FlowAgentBuilder':
        """Enhanced MCP config loading with automatic session management and full capability extraction"""
        if not MCP_AVAILABLE:
            wprint("MCP not available, skipping tool loading")
            return self

        if isinstance(config_path, dict):
            mcp_config = config_path
            from toolboxv2 import get_app
            name = self.config.name or "inline_config"
            path = Path(get_app().appdata) / "isaa" / "MCPConfig" / f"{name}.json"
            path.parent.mkdir(parents=True, exist_ok=True)
            path.write_text(json.dumps(mcp_config, indent=2))
            config_path = path
        else:
            config_path = Path(config_path)
            if not config_path.exists():
                raise FileNotFoundError(f"MCP config not found: {config_path}")

            try:
                with open(config_path, encoding='utf-8') as f:
                    if config_path.suffix.lower() in ['.yaml', '.yml']:
                        mcp_config = yaml.safe_load(f)
                    else:
                        mcp_config = json.load(f)

            except Exception as e:
                eprint(f"Failed to load MCP config from {config_path}: {e}")
                raise

        # Store config for async processing
        self._mcp_config_data = mcp_config
        self.config.mcp.config_path = str(config_path)

        # Mark for processing during build
        self._mcp_needs_loading = True

        iprint(f"MCP config loaded from {config_path}, will process during build")

        return self

    async def _process_mcp_config(self):
        """Process MCP configuration with proper task management"""
        if not hasattr(self, '_mcp_config_data') or not self._mcp_config_data:
            return

        mcp_config = self._mcp_config_data

        # Handle standard MCP server configuration with sequential processing to avoid task issues
        if 'mcpServers' in mcp_config:
            servers_to_load = []

            # Validate all servers first
            for server_name, server_config in mcp_config['mcpServers'].items():
                if self._validate_mcp_server_config(server_name, server_config):
                    servers_to_load.append((server_name, server_config))
                else:
                    wprint(f"Skipping invalid MCP server config: {server_name}")

            if servers_to_load:
                iprint(f"Processing {len(servers_to_load)} MCP servers sequentially...")

                # Process servers sequentially to avoid task boundary issues
                successful_loads = 0
                for server_name, server_config in servers_to_load:
                    try:
                        result = await asyncio.wait_for(
                            self._load_single_mcp_server(server_name, server_config),
                            timeout=5.0  # Per-server timeout
                        )

                        if result:
                            successful_loads += 1
                            iprint(f"✓ Successfully loaded MCP server: {server_name}")
                        else:
                            wprint(f"⚠ MCP server {server_name} loaded with issues")

                    except TimeoutError:
                        eprint(f"✗ MCP server {server_name} timed out after 15 seconds")
                    except Exception as e:
                        eprint(f"✗ Failed to load MCP server {server_name}: {e}")

                iprint(
                    f"MCP processing complete: {successful_loads}/{len(servers_to_load)} servers loaded successfully")

        # Handle direct tools configuration (legacy)
        elif 'tools' in mcp_config:
            for tool_config in mcp_config['tools']:
                try:
                    self._load_direct_mcp_tool(tool_config)
                except Exception as e:
                    eprint(f"Failed to load direct MCP tool: {e}")

    async def _load_single_mcp_server(self, server_name: str, server_config: dict[str, Any]) -> bool:
        """Load a single MCP server with timeout and error handling"""
        try:
            iprint(f"🔄 Processing MCP server: {server_name}")

            # Get session with timeout
            session = await self._mcp_session_manager.get_session_with_timeout(server_name, server_config)
            if not session:
                eprint(f"✗ Failed to create session for MCP server: {server_name}")
                return False

            # Extract capabilities with timeout
            capabilities = await self._mcp_session_manager.extract_capabilities_with_timeout(session, server_name)
            if not any(capabilities.values()):
                wprint(f"⚠ No capabilities found for MCP server: {server_name}")
                return False

            # Create wrappers for all capabilities
            await self._create_capability_wrappers(server_name, capabilities, session)

            total_caps = sum(len(caps) for caps in capabilities.values())
            iprint(f"✓ Created {total_caps} capability wrappers for: {server_name}")

            return True

        except Exception as e:
            eprint(f"✗ Error loading MCP server {server_name}: {e}")
            return False

    async def _create_capability_wrappers(self, server_name: str, capabilities: dict, session: ClientSession):
        """Create wrappers for all capabilities with error handling"""

        # Create tool wrappers
        for tool_name, tool_info in capabilities['tools'].items():
            try:
                wrapper_name = f"{server_name}_{tool_name}"
                tool_wrapper = self._create_tool_wrapper(server_name, tool_name, tool_info, session)

                self._mcp_tools[wrapper_name] = {
                    'function': tool_wrapper,
                    'description': tool_info['description'],
                    'type': 'tool',
                    'server': server_name,
                    'original_name': tool_name,
                    'input_schema': tool_info.get('input_schema'),
                    'output_schema': tool_info.get('output_schema')
                }
            except Exception as e:
                eprint(f"Failed to create tool wrapper {tool_name}: {e}")

        # Create resource wrappers
        for resource_uri, resource_info in capabilities['resources'].items():
            try:
                safe_name = resource_info['name'].replace('/', '_').replace(':', '_')
                wrapper_name = f"{server_name}_resource_{safe_name}"
                resource_wrapper = self._create_resource_wrapper(server_name, resource_uri, resource_info, session)

                self._mcp_tools[wrapper_name] = {
                    'function': resource_wrapper,
                    'description': f"Read resource: {resource_info['description']}",
                    'type': 'resource',
                    'server': server_name,
                    'original_uri': resource_uri
                }
            except Exception as e:
                eprint(f"Failed to create resource wrapper {resource_uri}: {e}")

        # Create resource template wrappers
        for template_uri, template_info in capabilities['resource_templates'].items():
            try:
                safe_name = template_info['name'].replace('/', '_').replace(':', '_')
                wrapper_name = f"{server_name}_template_{safe_name}"
                template_wrapper = self._create_resource_template_wrapper(server_name, template_uri, template_info,
                                                                          session)

                self._mcp_tools[wrapper_name] = {
                    'function': template_wrapper,
                    'description': f"Access resource template: {template_info['description']}",
                    'type': 'resource_template',
                    'server': server_name,
                    'original_template': template_uri
                }
            except Exception as e:
                eprint(f"Failed to create template wrapper {template_uri}: {e}")

        # Create prompt wrappers
        for prompt_name, prompt_info in capabilities['prompts'].items():
            try:
                wrapper_name = f"{server_name}_prompt_{prompt_name}"
                prompt_wrapper = self._create_prompt_wrapper(server_name, prompt_name, prompt_info, session)

                self._mcp_tools[wrapper_name] = {
                    'function': prompt_wrapper,
                    'description': f"Execute prompt: {prompt_info['description']}",
                    'type': 'prompt',
                    'server': server_name,
                    'original_name': prompt_name,
                    'arguments': prompt_info.get('arguments', [])
                }
            except Exception as e:
                eprint(f"Failed to create prompt wrapper {prompt_name}: {e}")

    @staticmethod
    def _validate_mcp_server_config(server_name: str, server_config: dict[str, Any]) -> bool:
        """Validate MCP server configuration"""
        command = server_config.get('command')
        if not command:
            eprint(f"MCP server {server_name} missing 'command' field")
            return False

        # Check if command exists and is executable
        if command in ['npx', 'node', 'python', 'python3', 'docker']:
            # These are common commands, assume they exist
            return True

        if server_config.get('transport') in ['http', 'streamable-http'] and server_config.get('url'):
            return True

        # For other commands, check if they exist
        import shutil
        if not shutil.which(command):
            wprint(f"MCP server {server_name}: command '{command}' not found in PATH")
            # Don't fail completely, just warn - the command might be available at runtime

        args = server_config.get('args', [])
        if not isinstance(args, list):
            eprint(f"MCP server {server_name}: 'args' must be a list")
            return False

        env = server_config.get('env', {})
        if not isinstance(env, dict):
            eprint(f"MCP server {server_name}: 'env' must be a dictionary")
            return False

        iprint(f"Validated MCP server config: {server_name}")
        return True

    def _load_direct_mcp_tool(self, tool_config: dict[str, Any]):
        """Load tool from direct configuration"""
        name = tool_config.get('name')
        description = tool_config.get('description', '')
        function_code = tool_config.get('function_code')

        if not name or not function_code:
            wprint(f"Incomplete tool config: {tool_config}")
            return

        # Create function from code
        try:
            namespace = {"__builtins__": __builtins__}
            exec(function_code, namespace)

            # Find the function
            func = None
            for obj in namespace.values():
                if callable(obj) and not getattr(obj, '__name__', '').startswith('_'):
                    func = obj
                    break

            if func:
                self._mcp_tools[name] = {
                    'function': func,
                    'description': description,
                    'source': 'code'
                }
                iprint(f"Loaded MCP tool from code: {name}")

        except Exception as e:
            eprint(f"Failed to load MCP tool {name}: {e}")

    def add_mcp_tool_from_code(self, name: str, code: str, description: str = "") -> 'FlowAgentBuilder':
        """Add MCP tool from code string"""
        tool_config = {
            'name': name,
            'description': description,
            'function_code': code
        }
        self._load_direct_mcp_tool(tool_config)
        return self

    # ===== A2A INTEGRATION =====

    def enable_a2a_server(self, host: str = "0.0.0.0", port: int = 5000,
                          agent_name: str = None, agent_description: str = None) -> 'FlowAgentBuilder':
        """Enable A2A server for agent-to-agent communication"""
        if not A2A_AVAILABLE:
            wprint("A2A not available, cannot enable server")
            return self

        self.config.a2a.enabled = True
        self.config.a2a.host = host
        self.config.a2a.port = port
        self.config.a2a.agent_name = agent_name or self.config.name
        self.config.a2a.agent_description = agent_description or self.config.description

        iprint(f"A2A server enabled: {host}:{port}")
        return self

    # ===== TELEMETRY INTEGRATION =====

    def enable_telemetry(self, service_name: str = None, endpoint: str = None,
                         console_export: bool = True) -> 'FlowAgentBuilder':
        """Enable OpenTelemetry tracing"""
        if not OTEL_AVAILABLE:
            wprint("OpenTelemetry not available, cannot enable telemetry")
            return self

        self.config.telemetry.enabled = True
        self.config.telemetry.service_name = service_name or self.config.name
        self.config.telemetry.endpoint = endpoint
        self.config.telemetry.console_export = console_export

        # Initialize tracer provider
        self._tracer_provider = TracerProvider()
        trace.set_tracer_provider(self._tracer_provider)

        # Add exporters
        if console_export:
            console_exporter = ConsoleSpanExporter()
            span_processor = BatchSpanProcessor(console_exporter)
            self._tracer_provider.add_span_processor(span_processor)

        if endpoint:
            try:
                otlp_exporter = OTLPSpanExporter(endpoint=endpoint)
                otlp_processor = BatchSpanProcessor(otlp_exporter)
                self._tracer_provider.add_span_processor(otlp_processor)
            except Exception as e:
                wprint(f"Failed to setup OTLP exporter: {e}")

        iprint(f"Telemetry enabled for service: {service_name}")
        return self

    # ===== CHECKPOINT CONFIGURATION =====

    def with_checkpointing(self, enabled: bool = True, interval_seconds: int = 300,
                           checkpoint_dir: str = "./checkpoints", max_checkpoints: int = 10) -> 'FlowAgentBuilder':
        """Configure checkpointing"""
        self.config.checkpoint.enabled = enabled
        self.config.checkpoint.interval_seconds = interval_seconds
        self.config.checkpoint.checkpoint_dir = checkpoint_dir
        self.config.checkpoint.max_checkpoints = max_checkpoints

        if enabled:
            # Ensure checkpoint directory exists
            Path(checkpoint_dir).mkdir(parents=True, exist_ok=True)
            iprint(f"Checkpointing enabled: {checkpoint_dir} (every {interval_seconds}s)")

        return self

    # ===== TOOL MANAGEMENT =====

    def add_tool(self, func: Callable, name: str = None, description: str = None) -> 'FlowAgentBuilder':
        """Add custom tool function"""
        tool_name = name or func.__name__
        self._custom_tools[tool_name] = (func, description or func.__doc__)

        iprint(f"Tool added: {tool_name}")
        return self

    def add_tools_from_module(self, module, prefix: str = "", exclude: list[str] = None) -> 'FlowAgentBuilder':
        """Add all functions from a module as tools"""
        exclude = exclude or []

        for name, obj in inspect.getmembers(module, inspect.isfunction):
            if name in exclude or name.startswith('_'):
                continue

            tool_name = f"{prefix}{name}" if prefix else name
            self.add_tool(obj, name=tool_name)

        iprint(f"Added tools from module {module.__name__}")
        return self

    # ===== PERSONA MANAGEMENT =====

    def add_persona_profile(self, profile_name: str, name: str, style: str = "professional",
                            tone: str = "friendly", personality_traits: list[str] = None,
                            custom_instructions: str = "", response_format: str = None,
                            text_length: str = None) -> 'FlowAgentBuilder':
        """Add a persona profile with optional format configuration"""

        if personality_traits is None:
            personality_traits = ["helpful", "concise"]

        # Create persona config
        persona_data = {
            "name": name,
            "style": style,
            "tone": tone,
            "personality_traits": personality_traits,
            "custom_instructions": custom_instructions,
            "apply_method": "system_prompt",
            "integration_level": "light"
        }

        # Add format config if specified
        if response_format or text_length:
            format_config = {
                "response_format": response_format or "frei-text",
                "text_length": text_length or "chat-conversation",
                "custom_instructions": "",
                "strict_format_adherence": True,
                "quality_threshold": 0.7
            }
            persona_data["format_config"] = format_config

        self.config.persona_profiles[profile_name] = persona_data
        iprint(f"Persona profile added: {profile_name}")
        return self

    def set_active_persona(self, profile_name: str) -> 'FlowAgentBuilder':
        """Set active persona profile"""
        if profile_name in self.config.persona_profiles:
            self.config.active_persona = profile_name
            iprint(f"Active persona set: {profile_name}")
        else:
            wprint(f"Persona profile not found: {profile_name}")
        return self

    def with_developer_persona(self, name: str = "Senior Developer") -> 'FlowAgentBuilder':
        """Add and set a pre-built developer persona"""
        return (self
                .add_persona_profile(
            "developer",
            name=name,
            style="technical",
            tone="professional",
            personality_traits=["precise", "thorough", "security_conscious", "best_practices"],
            custom_instructions="Focus on code quality, maintainability, and security. Always consider edge cases.",
            response_format="code-structure",
            text_length="detailed-indepth"
        )
                .set_active_persona("developer"))

    def with_analyst_persona(self, name: str = "Data Analyst") -> 'FlowAgentBuilder':
        """Add and set a pre-built analyst persona"""
        return (self
                .add_persona_profile(
            "analyst",
            name=name,
            style="analytical",
            tone="objective",
            personality_traits=["methodical", "insight_driven", "evidence_based"],
            custom_instructions="Focus on statistical rigor and actionable recommendations.",
            response_format="with-tables",
            text_length="detailed-indepth"
        )
                .set_active_persona("analyst"))

    def with_assistant_persona(self, name: str = "AI Assistant") -> 'FlowAgentBuilder':
        """Add and set a pre-built general assistant persona"""
        return (self
                .add_persona_profile(
            "assistant",
            name=name,
            style="friendly",
            tone="helpful",
            personality_traits=["helpful", "patient", "clear", "adaptive"],
            custom_instructions="Be helpful and adapt communication to user expertise level.",
            response_format="with-bullet-points",
            text_length="chat-conversation"
        )
                .set_active_persona("assistant"))

    def with_creative_persona(self, name: str = "Creative Assistant") -> 'FlowAgentBuilder':
        """Add and set a pre-built creative persona"""
        return (self
                .add_persona_profile(
            "creative",
            name=name,
            style="creative",
            tone="inspiring",
            personality_traits=["imaginative", "expressive", "innovative", "engaging"],
            custom_instructions="Think outside the box and provide creative, inspiring solutions.",
            response_format="md-text",
            text_length="detailed-indepth"
        )
                .set_active_persona("creative"))

    def with_executive_persona(self, name: str = "Executive Assistant") -> 'FlowAgentBuilder':
        """Add and set a pre-built executive persona"""
        return (self
                .add_persona_profile(
            "executive",
            name=name,
            style="professional",
            tone="authoritative",
            personality_traits=["strategic", "decisive", "results_oriented", "efficient"],
            custom_instructions="Provide strategic insights with executive-level clarity and focus on outcomes.",
            response_format="with-bullet-points",
            text_length="table-conversation"
        )
                .set_active_persona("executive"))

    # ===== VARIABLE MANAGEMENT =====

    def with_custom_variables(self, variables: dict[str, Any]) -> 'FlowAgentBuilder':
        """Add custom variables"""
        self.config.custom_variables.update(variables)
        return self

    def with_world_model(self, world_model: dict[str, Any]) -> 'FlowAgentBuilder':
        """Set initial world model"""
        self.config.initial_world_model.update(world_model)
        return self

    # ===== VALIDATION =====

    def validate_config(self) -> dict[str, list[str]]:
        """Validate the current configuration"""
        issues = {"errors": [], "warnings": []}

        # Validate required settings
        if not self.config.fast_llm_model:
            issues["errors"].append("Fast LLM model not specified")
        if not self.config.complex_llm_model:
            issues["errors"].append("Complex LLM model not specified")

        # Validate MCP configuration
        if self.config.mcp.enabled and not MCP_AVAILABLE:
            issues["errors"].append("MCP enabled but MCP not available")

        # Validate A2A configuration
        if self.config.a2a.enabled and not A2A_AVAILABLE:
            issues["errors"].append("A2A enabled but A2A not available")

        # Validate telemetry
        if self.config.telemetry.enabled and not OTEL_AVAILABLE:
            issues["errors"].append("Telemetry enabled but OpenTelemetry not available")

        # Validate personas
        if self.config.active_persona and self.config.active_persona not in self.config.persona_profiles:
            issues["errors"].append(f"Active persona '{self.config.active_persona}' not found in profiles")

        # Validate checkpoint directory
        if self.config.checkpoint.enabled:
            try:
                Path(self.config.checkpoint.checkpoint_dir).mkdir(parents=True, exist_ok=True)
            except Exception as e:
                issues["warnings"].append(f"Cannot create checkpoint directory: {e}")

        return issues

    # ===== MAIN BUILD METHOD =====

    async def build(self) -> FlowAgent:
        """Build the production-ready FlowAgent"""

        with Spinner(message=f"Building Agent {self.config.name}", symbols='c'):
            iprint(f"Building production FlowAgent: {self.config.name}")

            # Validate configuration
            validation_issues = self.validate_config()
            if validation_issues["errors"]:
                error_msg = f"Configuration validation failed: {', '.join(validation_issues['errors'])}"
                eprint(error_msg)
                raise ValueError(error_msg)

            # Log warnings
            for warning in validation_issues["warnings"]:
                wprint(f"Configuration warning: {warning}")

            try:
                # 1. Setup API configuration
                api_key = None
                if self.config.api_key_env_var:
                    api_key = os.getenv(self.config.api_key_env_var)
                    if not api_key:
                        wprint(f"API key env var {self.config.api_key_env_var} not set")

                # 2. Create persona if configured
                active_persona = None
                if self.config.active_persona and self.config.active_persona in self.config.persona_profiles:
                    persona_data = self.config.persona_profiles[self.config.active_persona]

                    # Create FormatConfig if present
                    format_config = None
                    if "format_config" in persona_data:
                        fc_data = persona_data.pop("format_config")
                        format_config = FormatConfig(
                            response_format=ResponseFormat(fc_data.get("response_format", "frei-text")),
                            text_length=TextLength(fc_data.get("text_length", "chat-conversation")),
                            custom_instructions=fc_data.get("custom_instructions", ""),
                            strict_format_adherence=fc_data.get("strict_format_adherence", True),
                            quality_threshold=fc_data.get("quality_threshold", 0.7)
                        )

                    active_persona = PersonaConfig(**persona_data)
                    active_persona.format_config = format_config

                    iprint(f"Using persona: {active_persona.name}")

                # 3. Create AgentModelData
                amd = AgentModelData(
                    name=self.config.name,
                    fast_llm_model=self.config.fast_llm_model,
                    complex_llm_model=self.config.complex_llm_model,
                    system_message=self.config.system_message,
                    temperature=self.config.temperature,
                    max_tokens=self.config.max_tokens_output,
                    max_input_tokens=self.config.max_tokens_input,
                    api_key=api_key,
                    budget_manager=self._budget_manager,
                    persona=active_persona,
                    use_fast_response=self.config.use_fast_response
                )

                # 4. Create FlowAgent
                agent = FlowAgent(
                    amd=amd,
                    world_model=self.config.initial_world_model.copy(),
                    verbose=self.config.verbose_logging,
                    enable_pause_resume=self.config.checkpoint.enabled,
                    checkpoint_interval=self.config.checkpoint.interval_seconds,
                    max_parallel_tasks=self.config.max_parallel_tasks
                )

                # 5. Add custom variables
                for key, value in self.config.custom_variables.items():
                    agent.set_variable(key, value)

                # 6. Add custom tools
                tools_added = 0
                for tool_name, (tool_func, tool_description) in self._custom_tools.items():
                    try:
                        await agent.add_tool(tool_func, tool_name, tool_description)
                        tools_added += 1
                    except Exception as e:
                        eprint(f"Failed to add tool {tool_name}: {e}")

                with Spinner(message="Loading MCP", symbols='w'):
                    # 6a. Process MCP configuration if needed
                    if hasattr(self, '_mcp_needs_loading') and self._mcp_needs_loading:
                        await self._process_mcp_config()

                # 7. Add MCP tools
                for tool_name, tool_info in self._mcp_tools.items():
                    try:
                        await agent.add_tool(
                            tool_info['function'],
                            tool_name,
                            tool_info['description']
                        )
                        tools_added += 1
                    except Exception as e:
                        eprint(f"Failed to add MCP tool {tool_name}: {e}")

                agent._mcp_session_manager = self._mcp_session_manager

                # 8. Setup MCP server
                if self.config.mcp.enabled and MCP_AVAILABLE:
                    try:
                        agent.setup_mcp_server(
                            host=self.config.mcp.host,
                            port=self.config.mcp.port,
                            name=self.config.mcp.server_name
                        )
                        iprint("MCP server configured")
                    except Exception as e:
                        eprint(f"Failed to setup MCP server: {e}")

                # 9. Setup A2A server
                if self.config.a2a.enabled and A2A_AVAILABLE:
                    try:
                        agent.setup_a2a_server(
                            host=self.config.a2a.host,
                            port=self.config.a2a.port
                        )
                        iprint("A2A server configured")
                    except Exception as e:
                        eprint(f"Failed to setup A2A server: {e}")

                # 10. Initialize enhanced session context
                try:
                    await agent.initialize_session_context(max_history=200)
                    iprint("Enhanced session context initialized")
                except Exception as e:
                    wprint(f"Session context initialization failed: {e}")


                # Final summary
                iprint("ok FlowAgent built successfully!")
                iprint(f"   Agent: {agent.amd.name}")
                iprint(f"   Tools: {tools_added}")
                iprint(f"   MCP: {'ok' if self.config.mcp.enabled else 'F'}")
                iprint(f"   A2A: {'ok' if self.config.a2a.enabled else 'F'}")
                iprint(f"   Telemetry: {'ok' if self.config.telemetry.enabled else 'F'}")
                iprint(f"   Checkpoints: {'ok' if self.config.checkpoint.enabled else 'F'}")
                iprint(f"   Persona: {active_persona.name if active_persona else 'Default'}")

                return agent

            except Exception as e:
                eprint(f"Failed to build FlowAgent: {e}")
                raise

    # ===== FACTORY METHODS =====

    @classmethod
    def create_developer_agent(cls, name: str = "DeveloperAgent",
                               with_mcp: bool = True, with_a2a: bool = False) -> 'FlowAgentBuilder':
        """Create a pre-configured developer agent"""
        builder = (cls()
                   .with_name(name)
                   .with_developer_persona()
                   .with_checkpointing(enabled=True, interval_seconds=300)
                   .verbose(True))

        if with_mcp:
            builder.enable_mcp_server(port=8001)
        if with_a2a:
            builder.enable_a2a_server(port=5001)

        return builder

    @classmethod
    def create_analyst_agent(cls, name: str = "AnalystAgent",
                             with_telemetry: bool = True) -> 'FlowAgentBuilder':
        """Create a pre-configured data analyst agent"""
        builder = (cls()
                   .with_name(name)
                   .with_analyst_persona()
                   .with_checkpointing(enabled=True)
                   .verbose(False))

        if with_telemetry:
            builder.enable_telemetry(console_export=True)

        return builder

    @classmethod
    def create_general_assistant(cls, name: str = "AssistantAgent",
                                 full_integration: bool = True) -> 'FlowAgentBuilder':
        """Create a general-purpose assistant with full integration"""
        builder = (cls()
                   .with_name(name)
                   .with_assistant_persona()
                   .with_checkpointing(enabled=True))

        if full_integration:
            builder.enable_mcp_server()
            builder.enable_a2a_server()
            builder.enable_telemetry()

        return builder

    @classmethod
    def create_creative_agent(cls, name: str = "CreativeAgent") -> 'FlowAgentBuilder':
        """Create a creative assistant agent"""
        return (cls()
                .with_name(name)
                .with_creative_persona()
                .with_temperature(0.8)  # More creative
                .with_checkpointing(enabled=True))

    @classmethod
    def create_executive_agent(cls, name: str = "ExecutiveAgent",
                               with_integrations: bool = True) -> 'FlowAgentBuilder':
        """Create an executive assistant agent"""
        builder = (cls()
                   .with_name(name)
                   .with_executive_persona()
                   .with_checkpointing(enabled=True))

        if with_integrations:
            builder.enable_a2a_server()  # Executives need A2A for delegation
            builder.enable_telemetry()  # Need metrics

        return builder

__init__(config=None, config_path=None)

Initialize builder with configuration

Source code in toolboxv2/mods/isaa/base/Agent/builder.py

def __init__(self, config: AgentConfig = None, config_path: str = None):
    """Initialize builder with configuration"""

    if config and config_path:
        raise ValueError("Provide either config object or config_path, not both")

    if config_path:
        self.config = self.load_config(config_path)
    elif config:
        self.config = config
    else:
        self.config = AgentConfig()

    # Runtime components
    self._custom_tools: dict[str, tuple[Callable, str]] = {}
    self._mcp_tools: dict[str, dict] = {}
    from toolboxv2.mods.isaa.extras.mcp_session_manager import MCPSessionManager

    self._mcp_session_manager = MCPSessionManager()

    self._budget_manager: BudgetManager = None
    self._tracer_provider: TracerProvider = None
    self._a2a_server: Any = None

    # Set logging level
    if self.config.verbose_logging:
        logging.getLogger().setLevel(logging.DEBUG)

    iprint(f"FlowAgent Builder initialized: {self.config.name}")

add_mcp_tool_from_code(name, code, description='')

Add MCP tool from code string

Source code in toolboxv2/mods/isaa/base/Agent/builder.py

def add_mcp_tool_from_code(self, name: str, code: str, description: str = "") -> 'FlowAgentBuilder':
    """Add MCP tool from code string"""
    tool_config = {
        'name': name,
        'description': description,
        'function_code': code
    }
    self._load_direct_mcp_tool(tool_config)
    return self

add_persona_profile(profile_name, name, style='professional', tone='friendly', personality_traits=None, custom_instructions='', response_format=None, text_length=None)

Add a persona profile with optional format configuration

Source code in toolboxv2/mods/isaa/base/Agent/builder.py

def add_persona_profile(self, profile_name: str, name: str, style: str = "professional",
                        tone: str = "friendly", personality_traits: list[str] = None,
                        custom_instructions: str = "", response_format: str = None,
                        text_length: str = None) -> 'FlowAgentBuilder':
    """Add a persona profile with optional format configuration"""

    if personality_traits is None:
        personality_traits = ["helpful", "concise"]

    # Create persona config
    persona_data = {
        "name": name,
        "style": style,
        "tone": tone,
        "personality_traits": personality_traits,
        "custom_instructions": custom_instructions,
        "apply_method": "system_prompt",
        "integration_level": "light"
    }

    # Add format config if specified
    if response_format or text_length:
        format_config = {
            "response_format": response_format or "frei-text",
            "text_length": text_length or "chat-conversation",
            "custom_instructions": "",
            "strict_format_adherence": True,
            "quality_threshold": 0.7
        }
        persona_data["format_config"] = format_config

    self.config.persona_profiles[profile_name] = persona_data
    iprint(f"Persona profile added: {profile_name}")
    return self

add_tool(func, name=None, description=None)

Add custom tool function

Source code in toolboxv2/mods/isaa/base/Agent/builder.py

def add_tool(self, func: Callable, name: str = None, description: str = None) -> 'FlowAgentBuilder':
    """Add custom tool function"""
    tool_name = name or func.__name__
    self._custom_tools[tool_name] = (func, description or func.__doc__)

    iprint(f"Tool added: {tool_name}")
    return self

add_tools_from_module(module, prefix='', exclude=None)

Add all functions from a module as tools

Source code in toolboxv2/mods/isaa/base/Agent/builder.py

def add_tools_from_module(self, module, prefix: str = "", exclude: list[str] = None) -> 'FlowAgentBuilder':
    """Add all functions from a module as tools"""
    exclude = exclude or []

    for name, obj in inspect.getmembers(module, inspect.isfunction):
        if name in exclude or name.startswith('_'):
            continue

        tool_name = f"{prefix}{name}" if prefix else name
        self.add_tool(obj, name=tool_name)

    iprint(f"Added tools from module {module.__name__}")
    return self

build() async

Build the production-ready FlowAgent

Source code in toolboxv2/mods/isaa/base/Agent/builder.py

async def build(self) -> FlowAgent:
    """Build the production-ready FlowAgent"""

    with Spinner(message=f"Building Agent {self.config.name}", symbols='c'):
        iprint(f"Building production FlowAgent: {self.config.name}")

        # Validate configuration
        validation_issues = self.validate_config()
        if validation_issues["errors"]:
            error_msg = f"Configuration validation failed: {', '.join(validation_issues['errors'])}"
            eprint(error_msg)
            raise ValueError(error_msg)

        # Log warnings
        for warning in validation_issues["warnings"]:
            wprint(f"Configuration warning: {warning}")

        try:
            # 1. Setup API configuration
            api_key = None
            if self.config.api_key_env_var:
                api_key = os.getenv(self.config.api_key_env_var)
                if not api_key:
                    wprint(f"API key env var {self.config.api_key_env_var} not set")

            # 2. Create persona if configured
            active_persona = None
            if self.config.active_persona and self.config.active_persona in self.config.persona_profiles:
                persona_data = self.config.persona_profiles[self.config.active_persona]

                # Create FormatConfig if present
                format_config = None
                if "format_config" in persona_data:
                    fc_data = persona_data.pop("format_config")
                    format_config = FormatConfig(
                        response_format=ResponseFormat(fc_data.get("response_format", "frei-text")),
                        text_length=TextLength(fc_data.get("text_length", "chat-conversation")),
                        custom_instructions=fc_data.get("custom_instructions", ""),
                        strict_format_adherence=fc_data.get("strict_format_adherence", True),
                        quality_threshold=fc_data.get("quality_threshold", 0.7)
                    )

                active_persona = PersonaConfig(**persona_data)
                active_persona.format_config = format_config

                iprint(f"Using persona: {active_persona.name}")

            # 3. Create AgentModelData
            amd = AgentModelData(
                name=self.config.name,
                fast_llm_model=self.config.fast_llm_model,
                complex_llm_model=self.config.complex_llm_model,
                system_message=self.config.system_message,
                temperature=self.config.temperature,
                max_tokens=self.config.max_tokens_output,
                max_input_tokens=self.config.max_tokens_input,
                api_key=api_key,
                budget_manager=self._budget_manager,
                persona=active_persona,
                use_fast_response=self.config.use_fast_response
            )

            # 4. Create FlowAgent
            agent = FlowAgent(
                amd=amd,
                world_model=self.config.initial_world_model.copy(),
                verbose=self.config.verbose_logging,
                enable_pause_resume=self.config.checkpoint.enabled,
                checkpoint_interval=self.config.checkpoint.interval_seconds,
                max_parallel_tasks=self.config.max_parallel_tasks
            )

            # 5. Add custom variables
            for key, value in self.config.custom_variables.items():
                agent.set_variable(key, value)

            # 6. Add custom tools
            tools_added = 0
            for tool_name, (tool_func, tool_description) in self._custom_tools.items():
                try:
                    await agent.add_tool(tool_func, tool_name, tool_description)
                    tools_added += 1
                except Exception as e:
                    eprint(f"Failed to add tool {tool_name}: {e}")

            with Spinner(message="Loading MCP", symbols='w'):
                # 6a. Process MCP configuration if needed
                if hasattr(self, '_mcp_needs_loading') and self._mcp_needs_loading:
                    await self._process_mcp_config()

            # 7. Add MCP tools
            for tool_name, tool_info in self._mcp_tools.items():
                try:
                    await agent.add_tool(
                        tool_info['function'],
                        tool_name,
                        tool_info['description']
                    )
                    tools_added += 1
                except Exception as e:
                    eprint(f"Failed to add MCP tool {tool_name}: {e}")

            agent._mcp_session_manager = self._mcp_session_manager

            # 8. Setup MCP server
            if self.config.mcp.enabled and MCP_AVAILABLE:
                try:
                    agent.setup_mcp_server(
                        host=self.config.mcp.host,
                        port=self.config.mcp.port,
                        name=self.config.mcp.server_name
                    )
                    iprint("MCP server configured")
                except Exception as e:
                    eprint(f"Failed to setup MCP server: {e}")

            # 9. Setup A2A server
            if self.config.a2a.enabled and A2A_AVAILABLE:
                try:
                    agent.setup_a2a_server(
                        host=self.config.a2a.host,
                        port=self.config.a2a.port
                    )
                    iprint("A2A server configured")
                except Exception as e:
                    eprint(f"Failed to setup A2A server: {e}")

            # 10. Initialize enhanced session context
            try:
                await agent.initialize_session_context(max_history=200)
                iprint("Enhanced session context initialized")
            except Exception as e:
                wprint(f"Session context initialization failed: {e}")


            # Final summary
            iprint("ok FlowAgent built successfully!")
            iprint(f"   Agent: {agent.amd.name}")
            iprint(f"   Tools: {tools_added}")
            iprint(f"   MCP: {'ok' if self.config.mcp.enabled else 'F'}")
            iprint(f"   A2A: {'ok' if self.config.a2a.enabled else 'F'}")
            iprint(f"   Telemetry: {'ok' if self.config.telemetry.enabled else 'F'}")
            iprint(f"   Checkpoints: {'ok' if self.config.checkpoint.enabled else 'F'}")
            iprint(f"   Persona: {active_persona.name if active_persona else 'Default'}")

            return agent

        except Exception as e:
            eprint(f"Failed to build FlowAgent: {e}")
            raise

create_analyst_agent(name='AnalystAgent', with_telemetry=True) classmethod

Create a pre-configured data analyst agent

Source code in toolboxv2/mods/isaa/base/Agent/builder.py

@classmethod
def create_analyst_agent(cls, name: str = "AnalystAgent",
                         with_telemetry: bool = True) -> 'FlowAgentBuilder':
    """Create a pre-configured data analyst agent"""
    builder = (cls()
               .with_name(name)
               .with_analyst_persona()
               .with_checkpointing(enabled=True)
               .verbose(False))

    if with_telemetry:
        builder.enable_telemetry(console_export=True)

    return builder

create_creative_agent(name='CreativeAgent') classmethod

Create a creative assistant agent

Source code in toolboxv2/mods/isaa/base/Agent/builder.py

@classmethod
def create_creative_agent(cls, name: str = "CreativeAgent") -> 'FlowAgentBuilder':
    """Create a creative assistant agent"""
    return (cls()
            .with_name(name)
            .with_creative_persona()
            .with_temperature(0.8)  # More creative
            .with_checkpointing(enabled=True))

create_developer_agent(name='DeveloperAgent', with_mcp=True, with_a2a=False) classmethod

Create a pre-configured developer agent

Source code in toolboxv2/mods/isaa/base/Agent/builder.py

@classmethod
def create_developer_agent(cls, name: str = "DeveloperAgent",
                           with_mcp: bool = True, with_a2a: bool = False) -> 'FlowAgentBuilder':
    """Create a pre-configured developer agent"""
    builder = (cls()
               .with_name(name)
               .with_developer_persona()
               .with_checkpointing(enabled=True, interval_seconds=300)
               .verbose(True))

    if with_mcp:
        builder.enable_mcp_server(port=8001)
    if with_a2a:
        builder.enable_a2a_server(port=5001)

    return builder

create_executive_agent(name='ExecutiveAgent', with_integrations=True) classmethod

Create an executive assistant agent

Source code in toolboxv2/mods/isaa/base/Agent/builder.py

@classmethod
def create_executive_agent(cls, name: str = "ExecutiveAgent",
                           with_integrations: bool = True) -> 'FlowAgentBuilder':
    """Create an executive assistant agent"""
    builder = (cls()
               .with_name(name)
               .with_executive_persona()
               .with_checkpointing(enabled=True))

    if with_integrations:
        builder.enable_a2a_server()  # Executives need A2A for delegation
        builder.enable_telemetry()  # Need metrics

    return builder

create_general_assistant(name='AssistantAgent', full_integration=True) classmethod

Create a general-purpose assistant with full integration

Source code in toolboxv2/mods/isaa/base/Agent/builder.py

@classmethod
def create_general_assistant(cls, name: str = "AssistantAgent",
                             full_integration: bool = True) -> 'FlowAgentBuilder':
    """Create a general-purpose assistant with full integration"""
    builder = (cls()
               .with_name(name)
               .with_assistant_persona()
               .with_checkpointing(enabled=True))

    if full_integration:
        builder.enable_mcp_server()
        builder.enable_a2a_server()
        builder.enable_telemetry()

    return builder

enable_a2a_server(host='0.0.0.0', port=5000, agent_name=None, agent_description=None)

Enable A2A server for agent-to-agent communication

Source code in toolboxv2/mods/isaa/base/Agent/builder.py

def enable_a2a_server(self, host: str = "0.0.0.0", port: int = 5000,
                      agent_name: str = None, agent_description: str = None) -> 'FlowAgentBuilder':
    """Enable A2A server for agent-to-agent communication"""
    if not A2A_AVAILABLE:
        wprint("A2A not available, cannot enable server")
        return self

    self.config.a2a.enabled = True
    self.config.a2a.host = host
    self.config.a2a.port = port
    self.config.a2a.agent_name = agent_name or self.config.name
    self.config.a2a.agent_description = agent_description or self.config.description

    iprint(f"A2A server enabled: {host}:{port}")
    return self

enable_mcp_server(host='0.0.0.0', port=8000, server_name=None)

Enable MCP server

Source code in toolboxv2/mods/isaa/base/Agent/builder.py

def enable_mcp_server(self, host: str = "0.0.0.0", port: int = 8000,
                      server_name: str = None) -> 'FlowAgentBuilder':
    """Enable MCP server"""
    if not MCP_AVAILABLE:
        wprint("MCP not available, cannot enable server")
        return self

    self.config.mcp.enabled = True
    self.config.mcp.host = host
    self.config.mcp.port = port
    self.config.mcp.server_name = server_name or f"{self.config.name}_MCP"

    iprint(f"MCP server enabled: {host}:{port}")
    return self

enable_telemetry(service_name=None, endpoint=None, console_export=True)

Enable OpenTelemetry tracing

Source code in toolboxv2/mods/isaa/base/Agent/builder.py

def enable_telemetry(self, service_name: str = None, endpoint: str = None,
                     console_export: bool = True) -> 'FlowAgentBuilder':
    """Enable OpenTelemetry tracing"""
    if not OTEL_AVAILABLE:
        wprint("OpenTelemetry not available, cannot enable telemetry")
        return self

    self.config.telemetry.enabled = True
    self.config.telemetry.service_name = service_name or self.config.name
    self.config.telemetry.endpoint = endpoint
    self.config.telemetry.console_export = console_export

    # Initialize tracer provider
    self._tracer_provider = TracerProvider()
    trace.set_tracer_provider(self._tracer_provider)

    # Add exporters
    if console_export:
        console_exporter = ConsoleSpanExporter()
        span_processor = BatchSpanProcessor(console_exporter)
        self._tracer_provider.add_span_processor(span_processor)

    if endpoint:
        try:
            otlp_exporter = OTLPSpanExporter(endpoint=endpoint)
            otlp_processor = BatchSpanProcessor(otlp_exporter)
            self._tracer_provider.add_span_processor(otlp_processor)
        except Exception as e:
            wprint(f"Failed to setup OTLP exporter: {e}")

    iprint(f"Telemetry enabled for service: {service_name}")
    return self

from_config_file(config_path) classmethod

Create builder from configuration file

Source code in toolboxv2/mods/isaa/base/Agent/builder.py

@classmethod
def from_config_file(cls, config_path: str) -> 'FlowAgentBuilder':
    """Create builder from configuration file"""
    return cls(config_path=config_path)

load_config(config_path)

Load agent configuration from file

Source code in toolboxv2/mods/isaa/base/Agent/builder.py

def load_config(self, config_path: str) -> AgentConfig:
    """Load agent configuration from file"""
    path = Path(config_path)
    if not path.exists():
        raise FileNotFoundError(f"Config file not found: {config_path}")

    try:
        with open(path, encoding='utf-8') as f:
            if path.suffix.lower() in ['.yaml', '.yml']:
                data = yaml.safe_load(f)
            else:
                data = json.load(f)

        return AgentConfig(**data)

    except Exception as e:
        eprint(f"Failed to load config from {config_path}: {e}")
        raise

load_mcp_tools_from_config(config_path)

Enhanced MCP config loading with automatic session management and full capability extraction

Source code in toolboxv2/mods/isaa/base/Agent/builder.py

def load_mcp_tools_from_config(self, config_path: str | dict) -> 'FlowAgentBuilder':
    """Enhanced MCP config loading with automatic session management and full capability extraction"""
    if not MCP_AVAILABLE:
        wprint("MCP not available, skipping tool loading")
        return self

    if isinstance(config_path, dict):
        mcp_config = config_path
        from toolboxv2 import get_app
        name = self.config.name or "inline_config"
        path = Path(get_app().appdata) / "isaa" / "MCPConfig" / f"{name}.json"
        path.parent.mkdir(parents=True, exist_ok=True)
        path.write_text(json.dumps(mcp_config, indent=2))
        config_path = path
    else:
        config_path = Path(config_path)
        if not config_path.exists():
            raise FileNotFoundError(f"MCP config not found: {config_path}")

        try:
            with open(config_path, encoding='utf-8') as f:
                if config_path.suffix.lower() in ['.yaml', '.yml']:
                    mcp_config = yaml.safe_load(f)
                else:
                    mcp_config = json.load(f)

        except Exception as e:
            eprint(f"Failed to load MCP config from {config_path}: {e}")
            raise

    # Store config for async processing
    self._mcp_config_data = mcp_config
    self.config.mcp.config_path = str(config_path)

    # Mark for processing during build
    self._mcp_needs_loading = True

    iprint(f"MCP config loaded from {config_path}, will process during build")

    return self

save_config(config_path, format='yaml')

Save current configuration to file

Source code in toolboxv2/mods/isaa/base/Agent/builder.py

def save_config(self, config_path: str, format: str = 'yaml'):
    """Save current configuration to file"""
    path = Path(config_path)
    path.parent.mkdir(parents=True, exist_ok=True)

    try:
        data = self.config.model_dump()

        with open(path, 'w', encoding='utf-8') as f:
            if format.lower() == 'yaml':
                yaml.dump(data, f, default_flow_style=False, indent=2)
            else:
                json.dump(data, f, indent=2)

        iprint(f"Configuration saved to {config_path}")

    except Exception as e:
        eprint(f"Failed to save config to {config_path}: {e}")
        raise

set_active_persona(profile_name)

Set active persona profile

Source code in toolboxv2/mods/isaa/base/Agent/builder.py

def set_active_persona(self, profile_name: str) -> 'FlowAgentBuilder':
    """Set active persona profile"""
    if profile_name in self.config.persona_profiles:
        self.config.active_persona = profile_name
        iprint(f"Active persona set: {profile_name}")
    else:
        wprint(f"Persona profile not found: {profile_name}")
    return self

validate_config()

Validate the current configuration

Source code in toolboxv2/mods/isaa/base/Agent/builder.py

def validate_config(self) -> dict[str, list[str]]:
    """Validate the current configuration"""
    issues = {"errors": [], "warnings": []}

    # Validate required settings
    if not self.config.fast_llm_model:
        issues["errors"].append("Fast LLM model not specified")
    if not self.config.complex_llm_model:
        issues["errors"].append("Complex LLM model not specified")

    # Validate MCP configuration
    if self.config.mcp.enabled and not MCP_AVAILABLE:
        issues["errors"].append("MCP enabled but MCP not available")

    # Validate A2A configuration
    if self.config.a2a.enabled and not A2A_AVAILABLE:
        issues["errors"].append("A2A enabled but A2A not available")

    # Validate telemetry
    if self.config.telemetry.enabled and not OTEL_AVAILABLE:
        issues["errors"].append("Telemetry enabled but OpenTelemetry not available")

    # Validate personas
    if self.config.active_persona and self.config.active_persona not in self.config.persona_profiles:
        issues["errors"].append(f"Active persona '{self.config.active_persona}' not found in profiles")

    # Validate checkpoint directory
    if self.config.checkpoint.enabled:
        try:
            Path(self.config.checkpoint.checkpoint_dir).mkdir(parents=True, exist_ok=True)
        except Exception as e:
            issues["warnings"].append(f"Cannot create checkpoint directory: {e}")

    return issues

verbose(enable=True)

Enable verbose logging

Source code in toolboxv2/mods/isaa/base/Agent/builder.py

def verbose(self, enable: bool = True) -> 'FlowAgentBuilder':
    """Enable verbose logging"""
    self.config.verbose_logging = enable
    if enable:
        logging.getLogger().setLevel(logging.DEBUG)
    return self

with_analyst_persona(name='Data Analyst')

Add and set a pre-built analyst persona

Source code in toolboxv2/mods/isaa/base/Agent/builder.py

def with_analyst_persona(self, name: str = "Data Analyst") -> 'FlowAgentBuilder':
    """Add and set a pre-built analyst persona"""
    return (self
            .add_persona_profile(
        "analyst",
        name=name,
        style="analytical",
        tone="objective",
        personality_traits=["methodical", "insight_driven", "evidence_based"],
        custom_instructions="Focus on statistical rigor and actionable recommendations.",
        response_format="with-tables",
        text_length="detailed-indepth"
    )
            .set_active_persona("analyst"))

with_assistant_persona(name='AI Assistant')

Add and set a pre-built general assistant persona

Source code in toolboxv2/mods/isaa/base/Agent/builder.py

def with_assistant_persona(self, name: str = "AI Assistant") -> 'FlowAgentBuilder':
    """Add and set a pre-built general assistant persona"""
    return (self
            .add_persona_profile(
        "assistant",
        name=name,
        style="friendly",
        tone="helpful",
        personality_traits=["helpful", "patient", "clear", "adaptive"],
        custom_instructions="Be helpful and adapt communication to user expertise level.",
        response_format="with-bullet-points",
        text_length="chat-conversation"
    )
            .set_active_persona("assistant"))

with_budget_manager(max_cost=10.0)

Enable budget management

Source code in toolboxv2/mods/isaa/base/Agent/builder.py

def with_budget_manager(self, max_cost: float = 10.0) -> 'FlowAgentBuilder':
    """Enable budget management"""
    if LITELLM_AVAILABLE:
        self._budget_manager = BudgetManager("agent")
        iprint(f"Budget manager enabled: ${max_cost}")
    else:
        wprint("LiteLLM not available, budget manager disabled")
    return self

with_checkpointing(enabled=True, interval_seconds=300, checkpoint_dir='./checkpoints', max_checkpoints=10)

Configure checkpointing

Source code in toolboxv2/mods/isaa/base/Agent/builder.py

def with_checkpointing(self, enabled: bool = True, interval_seconds: int = 300,
                       checkpoint_dir: str = "./checkpoints", max_checkpoints: int = 10) -> 'FlowAgentBuilder':
    """Configure checkpointing"""
    self.config.checkpoint.enabled = enabled
    self.config.checkpoint.interval_seconds = interval_seconds
    self.config.checkpoint.checkpoint_dir = checkpoint_dir
    self.config.checkpoint.max_checkpoints = max_checkpoints

    if enabled:
        # Ensure checkpoint directory exists
        Path(checkpoint_dir).mkdir(parents=True, exist_ok=True)
        iprint(f"Checkpointing enabled: {checkpoint_dir} (every {interval_seconds}s)")

    return self

with_creative_persona(name='Creative Assistant')

Add and set a pre-built creative persona

Source code in toolboxv2/mods/isaa/base/Agent/builder.py

def with_creative_persona(self, name: str = "Creative Assistant") -> 'FlowAgentBuilder':
    """Add and set a pre-built creative persona"""
    return (self
            .add_persona_profile(
        "creative",
        name=name,
        style="creative",
        tone="inspiring",
        personality_traits=["imaginative", "expressive", "innovative", "engaging"],
        custom_instructions="Think outside the box and provide creative, inspiring solutions.",
        response_format="md-text",
        text_length="detailed-indepth"
    )
            .set_active_persona("creative"))

with_custom_variables(variables)

Add custom variables

Source code in toolboxv2/mods/isaa/base/Agent/builder.py

def with_custom_variables(self, variables: dict[str, Any]) -> 'FlowAgentBuilder':
    """Add custom variables"""
    self.config.custom_variables.update(variables)
    return self

with_developer_persona(name='Senior Developer')

Add and set a pre-built developer persona

Source code in toolboxv2/mods/isaa/base/Agent/builder.py

def with_developer_persona(self, name: str = "Senior Developer") -> 'FlowAgentBuilder':
    """Add and set a pre-built developer persona"""
    return (self
            .add_persona_profile(
        "developer",
        name=name,
        style="technical",
        tone="professional",
        personality_traits=["precise", "thorough", "security_conscious", "best_practices"],
        custom_instructions="Focus on code quality, maintainability, and security. Always consider edge cases.",
        response_format="code-structure",
        text_length="detailed-indepth"
    )
            .set_active_persona("developer"))

with_executive_persona(name='Executive Assistant')

Add and set a pre-built executive persona

Source code in toolboxv2/mods/isaa/base/Agent/builder.py

def with_executive_persona(self, name: str = "Executive Assistant") -> 'FlowAgentBuilder':
    """Add and set a pre-built executive persona"""
    return (self
            .add_persona_profile(
        "executive",
        name=name,
        style="professional",
        tone="authoritative",
        personality_traits=["strategic", "decisive", "results_oriented", "efficient"],
        custom_instructions="Provide strategic insights with executive-level clarity and focus on outcomes.",
        response_format="with-bullet-points",
        text_length="table-conversation"
    )
            .set_active_persona("executive"))

with_models(fast_model, complex_model=None)

Set LLM models

Source code in toolboxv2/mods/isaa/base/Agent/builder.py

def with_models(self, fast_model: str, complex_model: str = None) -> 'FlowAgentBuilder':
    """Set LLM models"""
    self.config.fast_llm_model = fast_model
    if complex_model:
        self.config.complex_llm_model = complex_model
    return self

with_name(name)

Set agent name

Source code in toolboxv2/mods/isaa/base/Agent/builder.py

def with_name(self, name: str) -> 'FlowAgentBuilder':
    """Set agent name"""
    self.config.name = name
    return self

with_system_message(message)

Set system message

Source code in toolboxv2/mods/isaa/base/Agent/builder.py

def with_system_message(self, message: str) -> 'FlowAgentBuilder':
    """Set system message"""
    self.config.system_message = message
    return self

with_temperature(temp)

Set temperature

Source code in toolboxv2/mods/isaa/base/Agent/builder.py

def with_temperature(self, temp: float) -> 'FlowAgentBuilder':
    """Set temperature"""
    self.config.temperature = temp
    return self

with_world_model(world_model)

Set initial world model

Source code in toolboxv2/mods/isaa/base/Agent/builder.py

def with_world_model(self, world_model: dict[str, Any]) -> 'FlowAgentBuilder':
    """Set initial world model"""
    self.config.initial_world_model.update(world_model)
    return self

MCPConfig

Bases: BaseModel

MCP server and tools configuration

Source code in toolboxv2/mods/isaa/base/Agent/builder.py

class MCPConfig(BaseModel):
    """MCP server and tools configuration"""
    model_config = ConfigDict(arbitrary_types_allowed=True)

    enabled: bool = False
    config_path: Optional[str] = None  # Path to MCP tools config file
    server_name: Optional[str] = None
    host: str = "0.0.0.0"
    port: int = 8000
    auto_expose_tools: bool = True
    tools_from_config: list[dict[str, Any]] = Field(default_factory=list)

TelemetryConfig

Bases: BaseModel

OpenTelemetry configuration

Source code in toolboxv2/mods/isaa/base/Agent/builder.py

class TelemetryConfig(BaseModel):
    """OpenTelemetry configuration"""
    enabled: bool = False
    service_name: Optional[str] = None
    endpoint: Optional[str] = None  # OTLP endpoint
    console_export: bool = True
    batch_export: bool = True
    sample_rate: float = 1.0

detect_shell()

Detects the best available shell and the argument to execute a command. Returns: A tuple of (shell_executable, command_argument). e.g., ('/bin/bash', '-c') or ('powershell.exe', '-Command')

Source code in toolboxv2/mods/isaa/base/Agent/builder.py

def detect_shell() -> tuple[str, str]:
    """
    Detects the best available shell and the argument to execute a command.
    Returns:
        A tuple of (shell_executable, command_argument).
        e.g., ('/bin/bash', '-c') or ('powershell.exe', '-Command')
    """
    if platform.system() == "Windows":
        if shell_path := shutil.which("pwsh"):
            return shell_path, "-Command"
        if shell_path := shutil.which("powershell"):
            return shell_path, "-Command"
        return "cmd.exe", "/c"

    shell_env = os.environ.get("SHELL")
    if shell_env and shutil.which(shell_env):
        return shell_env, "-c"

    for shell in ["bash", "zsh", "sh"]:
        if shell_path := shutil.which(shell):
            return shell_path, "-c"

    return "/bin/sh", "-c"

example_production_usage() async

Production usage example with full features

Source code in toolboxv2/mods/isaa/base/Agent/builder.py

async def example_production_usage():
    """Production usage example with full features"""

    iprint("=== Production FlowAgent Builder Example ===")

    # Example 1: Developer agent with full MCP integration
    iprint("Creating developer agent with MCP integration...")

    # Add a custom tool
    def get_system_info():
        """Get basic system information"""
        import platform
        return {
            "platform": platform.platform(),
            "python_version": platform.python_version(),
            "architecture": platform.architecture()
        }

    developer_agent = await (FlowAgentBuilder
                             .create_developer_agent("ProductionDev", with_mcp=True, with_a2a=True)
                             .add_tool(get_system_info, "get_system_info", "Get system information")
                             .enable_telemetry(console_export=True)
                             .with_custom_variables({
        "project_name": "FlowAgent Production",
        "environment": "production"
    })
                             .build())

    # Test the developer agent
    dev_response = await developer_agent.a_run(
        "Hello! I'm working on {{ project_name }}. Can you tell me about the system and create a simple Python function?"
    )
    iprint(f"Developer agent response: {dev_response[:200]}...")

    # Example 2: Load from configuration file
    iprint("\nTesting configuration save/load...")

    # Save current config
    config_path = "/tmp/production_agent_config.yaml"
    builder = FlowAgentBuilder.create_analyst_agent("ConfigTestAgent")
    builder.save_config(config_path)

    # Load from config
    loaded_builder = FlowAgentBuilder.from_config_file(config_path)
    config_agent = await loaded_builder.build()

    config_response = await config_agent.a_run("Analyze this data: [1, 2, 3, 4, 5]")
    iprint(f"Config-loaded agent response: {config_response[:150]}...")

    # Example 3: Agent with MCP tools from config
    iprint("\nTesting MCP tools integration...")

    # Create a sample MCP config
    mcp_config = {
        "tools": [
            {
                "name": "weather_checker",
                "description": "Check weather for a location",
                "function_code": '''
async def weather_checker(location: str) -> str:
    """Mock weather checker"""
    import random
    conditions = ["sunny", "cloudy", "rainy", "snowy"]
    temp = random.randint(-10, 35)
    condition = random.choice(conditions)
    return f"Weather in {location}: {condition}, {temp}°C"
'''
            }
        ]
    }

    mcp_config_path = "/tmp/mcp_tools_config.json"
    with open(mcp_config_path, 'w') as f:
        json.dump(mcp_config, f, indent=2)

    mcp_agent = await (FlowAgentBuilder()
                       .with_name("MCPTestAgent")
                       .with_assistant_persona()
                       .enable_mcp_server(port=8002)
                       .load_mcp_tools_from_config(mcp_config_path)
                       .build())

    mcp_response = await mcp_agent.a_run("What's the weather like in Berlin?")
    iprint(f"MCP agent response: {mcp_response[:150]}...")

    # Show agent status
    iprint("\n=== Agent Status ===")
    status = developer_agent.status(pretty_print=False)
    iprint(f"Developer agent tools: {len(status['capabilities']['tool_names'])}")
    iprint(f"MCP agent tools: {len(mcp_agent.shared.get('available_tools', []))}")

    # Cleanup
    await developer_agent.close()
    await config_agent.close()
    await mcp_agent.close()

    iprint("Production example completed successfully!")

example_quick_start() async

Quick start examples for common scenarios

Source code in toolboxv2/mods/isaa/base/Agent/builder.py

async def example_quick_start():
    """Quick start examples for common scenarios"""

    iprint("=== Quick Start Examples ===")

    # 1. Simple developer agent
    dev_agent = await FlowAgentBuilder.create_developer_agent("QuickDev").build()
    response1 = await dev_agent.a_run("Create a Python function to validate email addresses")
    iprint(f"Quick dev response: {response1[:100]}...")
    await dev_agent.close()

    # 2. Analyst with custom data
    analyst_agent = await (FlowAgentBuilder
                           .create_analyst_agent("QuickAnalyst")
                           .with_custom_variables({"dataset": "sales_data_2024"})
                           .build())
    response2 = await analyst_agent.a_run("Analyze the trends in {{ dataset }}")
    iprint(f"Quick analyst response: {response2[:100]}...")
    await analyst_agent.close()

    # 3. Creative assistant
    creative_agent = await FlowAgentBuilder.create_creative_agent("QuickCreative").build()
    response3 = await creative_agent.a_run("Write a creative story about AI agents collaborating")
    iprint(f"Quick creative response: {response3[:100]}...")
    await creative_agent.close()

    iprint("Quick start examples completed!")

chain

CF

Chain Format - handles formatting and data extraction between tasks.

Source code in toolboxv2/mods/isaa/base/Agent/chain.py

class CF:
    """Chain Format - handles formatting and data extraction between tasks."""

    def __init__(self, format_class: type[BaseModel]):
        self.format_class = format_class
        self.extract_key: str | tuple | None = None
        self.is_parallel_extraction = False

    def __sub__(self, key: str | tuple):
        """Implements the - operator for data extraction keys."""
        new_cf = copy.copy(self)
        if isinstance(key, str):
            if '[n]' in key:
                new_cf.extract_key = key.replace('[n]', '')
                new_cf.is_parallel_extraction = True
            else:
                new_cf.extract_key = key
        elif isinstance(key, tuple):
            new_cf.extract_key = key
        return new_cf

__sub__(key)

Implements the - operator for data extraction keys.

Source code in toolboxv2/mods/isaa/base/Agent/chain.py

def __sub__(self, key: str | tuple):
    """Implements the - operator for data extraction keys."""
    new_cf = copy.copy(self)
    if isinstance(key, str):
        if '[n]' in key:
            new_cf.extract_key = key.replace('[n]', '')
            new_cf.is_parallel_extraction = True
        else:
            new_cf.extract_key = key
    elif isinstance(key, tuple):
        new_cf.extract_key = key
    return new_cf

Chain

Bases: ChainBase

The main class for creating and executing sequential chains of tasks.

Source code in toolboxv2/mods/isaa/base/Agent/chain.py

class Chain(ChainBase):
    """The main class for creating and executing sequential chains of tasks."""

    def __init__(self, agent: 'FlowAgent' = None):
        self.tasks: list[Any] = [agent] if agent else []
        self.progress_tracker: ChainPrinter | None = None

    @classmethod
    def _create_chain(cls, components: list[Any]) -> 'Chain':
        chain = cls()
        chain.tasks = components
        return chain

    def _extract_data(self, data: dict, cf: CF) -> Any:
        """Extracts data from a dictionary based on the CF configuration."""
        if not isinstance(data, dict):
            return data

        key = cf.extract_key
        if key == '*':
            return data
        if isinstance(key, tuple):
            return {k: data.get(k) for k in key if k in data}
        if isinstance(key, str) and key in data:
            return data[key]
        return data  # Return original data if key not found

    async def a_run(self, query: Any, **kwargs):
        """
        Executes the chain of tasks asynchronously with dynamic method selection,
        data extraction, and auto-parallelization.
        """
        current_data = query

        # We need to iterate with an index to look ahead
        i = 0
        while i < len(self.tasks):
            task = self.tasks[i]

            # --- Auto-Erkennung und Ausführung ---
            if hasattr(task, 'a_run') and hasattr(task, 'a_format_class'):
                next_task = self.tasks[i + 1] if (i + 1) < len(self.tasks) else None
                task.active_session = kwargs.get("session_id", "default")
                # Dynamische Entscheidung: a_format_class oder a_run aufrufen?
                if isinstance(next_task, CF):
                    # Nächste Aufgabe ist Formatierung, also a_format_class aufrufen
                    current_data = await task.a_format_class(
                        next_task.format_class, str(current_data), **kwargs
                    )
                else:
                    # Standardausführung
                    current_data = await task.a_run(str(current_data), **kwargs)
                task.active_session = None

            elif isinstance(task, CF):
                # --- Auto-Extraktion und Parallelisierung ---
                if task.extract_key:
                    extracted_data = self._extract_data(current_data, task)

                    if task.is_parallel_extraction and isinstance(extracted_data, list):
                        next_task_for_parallel = self.tasks[i + 1] if (i + 1) < len(self.tasks) else None
                        if next_task_for_parallel:
                            # Erstelle eine temporäre Parallel-Kette und führe sie aus
                            parallel_runner = ParallelChain([next_task_for_parallel] * len(extracted_data))

                            # Führe jeden Task mit dem entsprechenden Datenelement aus
                            parallel_tasks = [
                                next_task_for_parallel.a_run(item, **kwargs) for item in extracted_data
                            ]
                            current_data = await asyncio.gather(*parallel_tasks)

                            print("Parallel results:", type(current_data))
                            print("Parallel results:", len(current_data))
                            # Überspringe die nächste Aufgabe, da sie bereits parallel ausgeführt wurde
                            i += 1
                        else:
                            current_data = extracted_data
                    else:
                        current_data = extracted_data
                else:
                    # Keine Extraktion, Daten bleiben unverändert (CF dient nur als Marker)
                    pass

            elif isinstance(task, ParallelChain | ConditionalChain | ErrorHandlingChain):
                current_data = await task.a_run(current_data, **kwargs)

            elif callable(task) and not isinstance(task, (ChainBase, type)):
                # Check if the function is async, then await it
                if asyncio.iscoroutinefunction(task):
                    current_data = await task(current_data)
                # Otherwise, run the synchronous function normally
                else:
                    current_data = task(current_data)
            elif hasattr(task, 'a_run'):
                current_data = await task.a_run(current_data, **kwargs)
            elif isinstance(task, IS):
                # IS needs to be paired with >> to form a ConditionalChain
                next_task_for_cond = self.tasks[i + 1] if (i + 1) < len(self.tasks) else None
                if next_task_for_cond:
                    # Form a conditional chain on the fly
                    conditional_task = ConditionalChain(task, next_task_for_cond)
                    # Check for a false branch defined with %
                    next_next_task = self.tasks[i + 2] if (i + 2) < len(self.tasks) else None
                    if isinstance(next_next_task, ConditionalChain) and next_next_task.false_branch:
                        conditional_task.false_branch = next_next_task.false_branch
                        i += 1  # also skip the false branch marker

                    current_data = await conditional_task.a_run(current_data, **kwargs)
                    i += 1  # Skip the next task as it's part of the conditional
                else:
                    raise ValueError("IS condition must be followed by a task to execute.")

            i += 1  # Gehe zur nächsten Aufgabe

        return current_data

a_run(query, **kwargs) async

Executes the chain of tasks asynchronously with dynamic method selection, data extraction, and auto-parallelization.

Source code in toolboxv2/mods/isaa/base/Agent/chain.py

async def a_run(self, query: Any, **kwargs):
    """
    Executes the chain of tasks asynchronously with dynamic method selection,
    data extraction, and auto-parallelization.
    """
    current_data = query

    # We need to iterate with an index to look ahead
    i = 0
    while i < len(self.tasks):
        task = self.tasks[i]

        # --- Auto-Erkennung und Ausführung ---
        if hasattr(task, 'a_run') and hasattr(task, 'a_format_class'):
            next_task = self.tasks[i + 1] if (i + 1) < len(self.tasks) else None
            task.active_session = kwargs.get("session_id", "default")
            # Dynamische Entscheidung: a_format_class oder a_run aufrufen?
            if isinstance(next_task, CF):
                # Nächste Aufgabe ist Formatierung, also a_format_class aufrufen
                current_data = await task.a_format_class(
                    next_task.format_class, str(current_data), **kwargs
                )
            else:
                # Standardausführung
                current_data = await task.a_run(str(current_data), **kwargs)
            task.active_session = None

        elif isinstance(task, CF):
            # --- Auto-Extraktion und Parallelisierung ---
            if task.extract_key:
                extracted_data = self._extract_data(current_data, task)

                if task.is_parallel_extraction and isinstance(extracted_data, list):
                    next_task_for_parallel = self.tasks[i + 1] if (i + 1) < len(self.tasks) else None
                    if next_task_for_parallel:
                        # Erstelle eine temporäre Parallel-Kette und führe sie aus
                        parallel_runner = ParallelChain([next_task_for_parallel] * len(extracted_data))

                        # Führe jeden Task mit dem entsprechenden Datenelement aus
                        parallel_tasks = [
                            next_task_for_parallel.a_run(item, **kwargs) for item in extracted_data
                        ]
                        current_data = await asyncio.gather(*parallel_tasks)

                        print("Parallel results:", type(current_data))
                        print("Parallel results:", len(current_data))
                        # Überspringe die nächste Aufgabe, da sie bereits parallel ausgeführt wurde
                        i += 1
                    else:
                        current_data = extracted_data
                else:
                    current_data = extracted_data
            else:
                # Keine Extraktion, Daten bleiben unverändert (CF dient nur als Marker)
                pass

        elif isinstance(task, ParallelChain | ConditionalChain | ErrorHandlingChain):
            current_data = await task.a_run(current_data, **kwargs)

        elif callable(task) and not isinstance(task, (ChainBase, type)):
            # Check if the function is async, then await it
            if asyncio.iscoroutinefunction(task):
                current_data = await task(current_data)
            # Otherwise, run the synchronous function normally
            else:
                current_data = task(current_data)
        elif hasattr(task, 'a_run'):
            current_data = await task.a_run(current_data, **kwargs)
        elif isinstance(task, IS):
            # IS needs to be paired with >> to form a ConditionalChain
            next_task_for_cond = self.tasks[i + 1] if (i + 1) < len(self.tasks) else None
            if next_task_for_cond:
                # Form a conditional chain on the fly
                conditional_task = ConditionalChain(task, next_task_for_cond)
                # Check for a false branch defined with %
                next_next_task = self.tasks[i + 2] if (i + 2) < len(self.tasks) else None
                if isinstance(next_next_task, ConditionalChain) and next_next_task.false_branch:
                    conditional_task.false_branch = next_next_task.false_branch
                    i += 1  # also skip the false branch marker

                current_data = await conditional_task.a_run(current_data, **kwargs)
                i += 1  # Skip the next task as it's part of the conditional
            else:
                raise ValueError("IS condition must be followed by a task to execute.")

        i += 1  # Gehe zur nächsten Aufgabe

    return current_data

ChainBase

Abstract base class for all chain types, providing common operators.

Source code in toolboxv2/mods/isaa/base/Agent/chain.py

class ChainBase:
    """Abstract base class for all chain types, providing common operators."""

    def __rshift__(self, other: Any) -> 'Chain':
        """Implements the >> operator to chain tasks sequentially."""
        if isinstance(self, Chain):
            new_tasks = self.tasks + [other]
            return Chain._create_chain(new_tasks)
        return Chain._create_chain([self, other])

    def __add__(self, other: Any) -> 'ParallelChain':
        """Implements the + operator for parallel execution."""
        return ParallelChain([self, other])

    def __and__(self, other: Any) -> 'ParallelChain':
        """Implements the & operator, an alias for parallel execution."""
        return ParallelChain([self, other])

    def __or__(self, other: Any) -> 'ErrorHandlingChain':
        """Implements the | operator for defining a fallback/error handling path."""
        return ErrorHandlingChain(self, other)

    def __mod__(self, other: Any) -> 'ConditionalChain':
        """Implements the % operator for defining a false/else branch in a condition."""
        # This is typically used after a conditional chain.
        if isinstance(self, ConditionalChain):
            self.false_branch = other
            return self
        # Allows creating a conditional chain directly
        return ConditionalChain(None, self, other)

    def set_progress_callback(self, progress_tracker: 'ProgressTracker'):
        """Recursively sets the progress callback for all tasks in the chain."""
        tasks_to_process = []
        if hasattr(self, 'tasks'): tasks_to_process.extend(self.tasks)  # Chain
        if hasattr(self, 'agents'): tasks_to_process.extend(self.agents)  # ParallelChain
        if hasattr(self, 'true_branch'): tasks_to_process.append(self.true_branch)  # ConditionalChain
        if hasattr(self, 'false_branch') and self.false_branch: tasks_to_process.append(
            self.false_branch)  # ConditionalChain
        if hasattr(self, 'primary'): tasks_to_process.append(self.primary)  # ErrorHandlingChain
        if hasattr(self, 'fallback'): tasks_to_process.append(self.fallback)  # ErrorHandlingChain

        for task in tasks_to_process:
            if hasattr(task, 'set_progress_callback'):
                task.set_progress_callback(progress_tracker)

    def __call__(self, *args, **kwargs):
        """Allows the chain to be called like a function, returning an awaitable runner."""
        return self._Runner(self, args, kwargs)

    class _Runner:
        def __init__(self, parent, args, kwargs):
            self.parent = parent
            self.args = args
            self.kwargs = kwargs

        def __call__(self):
            """Synchronous execution."""
            return asyncio.run(self.parent.a_run(*self.args, **self.kwargs))

        def __await__(self):
            """Asynchronous execution."""
            return self.parent.a_run(*self.args, **self.kwargs).__await__()

__add__(other)

Implements the + operator for parallel execution.

Source code in toolboxv2/mods/isaa/base/Agent/chain.py

def __add__(self, other: Any) -> 'ParallelChain':
    """Implements the + operator for parallel execution."""
    return ParallelChain([self, other])

__and__(other)

Implements the & operator, an alias for parallel execution.

Source code in toolboxv2/mods/isaa/base/Agent/chain.py

def __and__(self, other: Any) -> 'ParallelChain':
    """Implements the & operator, an alias for parallel execution."""
    return ParallelChain([self, other])

__call__(*args, **kwargs)

Allows the chain to be called like a function, returning an awaitable runner.

Source code in toolboxv2/mods/isaa/base/Agent/chain.py

def __call__(self, *args, **kwargs):
    """Allows the chain to be called like a function, returning an awaitable runner."""
    return self._Runner(self, args, kwargs)

__mod__(other)

Implements the % operator for defining a false/else branch in a condition.

Source code in toolboxv2/mods/isaa/base/Agent/chain.py

def __mod__(self, other: Any) -> 'ConditionalChain':
    """Implements the % operator for defining a false/else branch in a condition."""
    # This is typically used after a conditional chain.
    if isinstance(self, ConditionalChain):
        self.false_branch = other
        return self
    # Allows creating a conditional chain directly
    return ConditionalChain(None, self, other)

__or__(other)

Implements the | operator for defining a fallback/error handling path.

Source code in toolboxv2/mods/isaa/base/Agent/chain.py

def __or__(self, other: Any) -> 'ErrorHandlingChain':
    """Implements the | operator for defining a fallback/error handling path."""
    return ErrorHandlingChain(self, other)

__rshift__(other)

Implements the >> operator to chain tasks sequentially.

Source code in toolboxv2/mods/isaa/base/Agent/chain.py

def __rshift__(self, other: Any) -> 'Chain':
    """Implements the >> operator to chain tasks sequentially."""
    if isinstance(self, Chain):
        new_tasks = self.tasks + [other]
        return Chain._create_chain(new_tasks)
    return Chain._create_chain([self, other])

set_progress_callback(progress_tracker)

Recursively sets the progress callback for all tasks in the chain.

Source code in toolboxv2/mods/isaa/base/Agent/chain.py

def set_progress_callback(self, progress_tracker: 'ProgressTracker'):
    """Recursively sets the progress callback for all tasks in the chain."""
    tasks_to_process = []
    if hasattr(self, 'tasks'): tasks_to_process.extend(self.tasks)  # Chain
    if hasattr(self, 'agents'): tasks_to_process.extend(self.agents)  # ParallelChain
    if hasattr(self, 'true_branch'): tasks_to_process.append(self.true_branch)  # ConditionalChain
    if hasattr(self, 'false_branch') and self.false_branch: tasks_to_process.append(
        self.false_branch)  # ConditionalChain
    if hasattr(self, 'primary'): tasks_to_process.append(self.primary)  # ErrorHandlingChain
    if hasattr(self, 'fallback'): tasks_to_process.append(self.fallback)  # ErrorHandlingChain

    for task in tasks_to_process:
        if hasattr(task, 'set_progress_callback'):
            task.set_progress_callback(progress_tracker)

ConditionalChain

Bases: ChainBase

Handles conditional execution based on a condition.

Source code in toolboxv2/mods/isaa/base/Agent/chain.py

class ConditionalChain(ChainBase):
    """Handles conditional execution based on a condition."""

    def __init__(self, condition: IS, true_branch: Any, false_branch: Any = None):
        self.condition = condition
        self.true_branch = true_branch
        self.false_branch = false_branch

    async def a_run(self, data: Any, **kwargs):
        """Executes the true or false branch based on the condition."""
        condition_met = False
        if isinstance(self.condition, IS) and isinstance(data, dict):
            if data.get(self.condition.key) == self.condition.expected_value:
                condition_met = True

        if condition_met:
            return await self.true_branch.a_run(data, **kwargs)
        elif self.false_branch:
            return await self.false_branch.a_run(data, **kwargs)
        return data  # Return original data if condition not met and no false branch

a_run(data, **kwargs) async

Executes the true or false branch based on the condition.

Source code in toolboxv2/mods/isaa/base/Agent/chain.py

async def a_run(self, data: Any, **kwargs):
    """Executes the true or false branch based on the condition."""
    condition_met = False
    if isinstance(self.condition, IS) and isinstance(data, dict):
        if data.get(self.condition.key) == self.condition.expected_value:
            condition_met = True

    if condition_met:
        return await self.true_branch.a_run(data, **kwargs)
    elif self.false_branch:
        return await self.false_branch.a_run(data, **kwargs)
    return data  # Return original data if condition not met and no false branch

ErrorHandlingChain

Bases: ChainBase

Handles exceptions in a primary chain by executing a fallback chain.

Source code in toolboxv2/mods/isaa/base/Agent/chain.py

class ErrorHandlingChain(ChainBase):
    """Handles exceptions in a primary chain by executing a fallback chain."""

    def __init__(self, primary: Any, fallback: Any):
        self.primary = primary
        self.fallback = fallback

    async def a_run(self, query: Any, **kwargs):
        """Tries the primary chain and executes the fallback on failure."""
        try:
            return await self.primary.a_run(query, **kwargs)
        except Exception as e:
            print(f"Primary chain failed with error: {e}. Running fallback.")
            return await self.fallback.a_run(query, **kwargs)

a_run(query, **kwargs) async

Tries the primary chain and executes the fallback on failure.

Source code in toolboxv2/mods/isaa/base/Agent/chain.py

async def a_run(self, query: Any, **kwargs):
    """Tries the primary chain and executes the fallback on failure."""
    try:
        return await self.primary.a_run(query, **kwargs)
    except Exception as e:
        print(f"Primary chain failed with error: {e}. Running fallback.")
        return await self.fallback.a_run(query, **kwargs)

Function

Bases: ChainBase

A wrapper to treat native Python functions as chainable components.

Source code in toolboxv2/mods/isaa/base/Agent/chain.py

class Function(ChainBase):
    """A wrapper to treat native Python functions as chainable components."""

    def __init__(self, func: callable):
        if not callable(func):
            raise TypeError("Function object must be initialized with a callable.")
        self.func = func
        # Get a meaningful name for visualization
        self.func_name = getattr(func, '__name__', 'anonymous_lambda')

    async def a_run(self, data: Any, **kwargs):
        """Executes the wrapped function, handling both sync and async cases."""
        # Note: kwargs from the chain run are not passed to the native function
        # to maintain a simple, predictable (data in -> data out) interface.
        if asyncio.iscoroutinefunction(self.func):
            return await self.func(data)
        else:
            return self.func(data)

    def __repr__(self):
        return f"Function(name='{self.func_name}')"

a_run(data, **kwargs) async

Executes the wrapped function, handling both sync and async cases.

Source code in toolboxv2/mods/isaa/base/Agent/chain.py

async def a_run(self, data: Any, **kwargs):
    """Executes the wrapped function, handling both sync and async cases."""
    # Note: kwargs from the chain run are not passed to the native function
    # to maintain a simple, predictable (data in -> data out) interface.
    if asyncio.iscoroutinefunction(self.func):
        return await self.func(data)
    else:
        return self.func(data)

IS

Conditional check for branching logic.

Source code in toolboxv2/mods/isaa/base/Agent/chain.py

class IS:
    """Conditional check for branching logic."""

    def __init__(self, key: str, expected_value: Any = True):
        self.key = key
        self.expected_value = expected_value

ParallelChain

Bases: ChainBase

Handles parallel execution of multiple agents or chains.

Source code in toolboxv2/mods/isaa/base/Agent/chain.py

class ParallelChain(ChainBase):
    """Handles parallel execution of multiple agents or chains."""

    def __init__(self, agents: list[Union['FlowAgent', ChainBase]]):
        self.agents = agents

    async def a_run(self, query: Any, **kwargs):
        """Runs all agents/chains in parallel."""
        tasks = [agent.a_run(query, **kwargs) for agent in self.agents]
        results = await asyncio.gather(*tasks)
        return self._combine_results(results)

    def _combine_results(self, results: list[Any]) -> Any:
        """Intelligently combines parallel results."""
        if all(isinstance(r, str) for r in results):
            return " | ".join(results)
        return results

a_run(query, **kwargs) async

Runs all agents/chains in parallel.

Source code in toolboxv2/mods/isaa/base/Agent/chain.py

async def a_run(self, query: Any, **kwargs):
    """Runs all agents/chains in parallel."""
    tasks = [agent.a_run(query, **kwargs) for agent in self.agents]
    results = await asyncio.gather(*tasks)
    return self._combine_results(results)

chain_to_graph(self)

Convert chain to hierarchical structure with complete component detection.

Source code in toolboxv2/mods/isaa/base/Agent/chain.py

def chain_to_graph(self) -> dict[str, Any]:
    """Convert chain to hierarchical structure with complete component detection."""

    def process_component(comp, depth=0, visited=None):
        if visited is None:
            visited = set()

        # Prevent infinite recursion
        comp_id = id(comp)
        if comp_id in visited or depth > 20:
            return {"type": "Circular", "display": "[CIRCULAR_REF]", "depth": depth}
        visited.add(comp_id)

        if comp is None:
            return {"type": "Error", "display": "[NULL]", "depth": depth}

        try:
            # Agent detection
            if hasattr(comp, 'amd') and comp.amd:
                return {
                    "type": "Agent",
                    "display": f"[Agent] {comp.amd.name}",
                    "name": comp.amd.name,
                    "depth": depth
                }

            # Format detection (CF) with parallel detection
            if hasattr(comp, 'format_class'):
                name = comp.format_class.__name__
                display = f"[Format] {name}"

                result = {
                    "type": "Format",
                    "display": display,
                    "format_class": name,
                    "extract_key": getattr(comp, 'extract_key', None),
                    "depth": depth,
                    "creates_parallel": False
                }

                # Extract key visualization
                if hasattr(comp, 'extract_key') and comp.extract_key:
                    key = comp.extract_key
                    if key == '*':
                        display += " \033[90m(*all*)\033[0m"
                    elif isinstance(key, str):
                        display += f" \033[90m(→{key})\033[0m"
                    elif isinstance(key, tuple):
                        display += f" \033[90m(→{','.join(key)})\033[0m"

                # Parallel detection
                if hasattr(comp, 'parallel_count') and comp.parallel_count == 'n':
                    display += " \033[95m[PARALLEL]\033[0m"
                    result["creates_parallel"] = True
                    result["parallel_type"] = "auto_n"

                result["display"] = display
                return result

            # Condition detection (IS)
            if hasattr(comp, 'key') and hasattr(comp, 'expected_value'):
                return {
                    "type": "Condition",
                    "display": f"[Condition] IS {comp.key}=='{comp.expected_value}'",
                    "condition_key": comp.key,
                    "expected_value": comp.expected_value,
                    "depth": depth
                }

            # Parallel chain detection
            if hasattr(comp, 'agents') and isinstance(comp.agents, list | tuple):
                branches = []
                for i, agent in enumerate(comp.agents):
                    if agent:
                        branch_data = process_component(agent, depth + 1, visited.copy())
                        branch_data["branch_id"] = i
                        branches.append(branch_data)

                return {
                    "type": "Parallel",
                    "display": f"[Parallel] {len(branches)} branches",
                    "branches": branches,
                    "branch_count": len(branches),
                    "execution_type": "concurrent",
                    "depth": depth
                }

            if isinstance(comp, Function):
                return {
                    "type": "Function",
                    "display": f"[Func] {comp.func_name}",
                    "function_name": comp.func_name,
                    "depth": depth
                }

            # Conditional chain detection
            if hasattr(comp, 'condition') and hasattr(comp, 'true_branch'):
                condition_data = process_component(comp.condition, depth + 1,
                                                   visited.copy()) if comp.condition else None
                true_data = process_component(comp.true_branch, depth + 1, visited.copy()) if comp.true_branch else None
                false_data = None

                if hasattr(comp, 'false_branch') and comp.false_branch:
                    false_data = process_component(comp.false_branch, depth + 1, visited.copy())

                return {
                    "type": "Conditional",
                    "display": "[Conditional] Branch Logic",
                    "condition": condition_data,
                    "true_branch": true_data,
                    "false_branch": false_data,
                    "has_false_branch": false_data is not None,
                    "depth": depth
                }

            # Error handling detection
            if hasattr(comp, 'primary') and hasattr(comp, 'fallback'):
                primary_data = process_component(comp.primary, depth + 1, visited.copy()) if comp.primary else None
                fallback_data = process_component(comp.fallback, depth + 1, visited.copy()) if comp.fallback else None

                return {
                    "type": "ErrorHandling",
                    "display": "[Try-Catch] Error Handler",
                    "primary": primary_data,
                    "fallback": fallback_data,
                    "has_fallback": fallback_data is not None,
                    "depth": depth
                }

            # Regular chain detection
            if hasattr(comp, 'tasks') and isinstance(comp.tasks, list | tuple):
                tasks = []
                for i, task in enumerate(comp.tasks):
                    if task is not None:
                        task_data = process_component(task, depth + 1, visited.copy())
                        task_data["task_id"] = i
                        tasks.append(task_data)

                # Analyze chain characteristics
                has_conditionals = any(t.get("type") == "Conditional" for t in tasks)
                has_parallels = any(t.get("type") == "Parallel" for t in tasks)
                has_error_handling = any(t.get("type") == "ErrorHandling" for t in tasks)
                has_auto_parallel = any(t.get("creates_parallel", False) for t in tasks)

                chain_type = "Sequential"
                if has_auto_parallel:
                    chain_type = "Auto-Parallel"
                elif has_conditionals and has_parallels:
                    chain_type = "Complex"
                elif has_conditionals:
                    chain_type = "Conditional"
                elif has_parallels:
                    chain_type = "Mixed-Parallel"
                elif has_error_handling:
                    chain_type = "Error-Handling"

                return {
                    "type": "Chain",
                    "display": f"[Chain] {chain_type}",
                    "tasks": tasks,
                    "task_count": len(tasks),
                    "chain_type": chain_type,
                    "has_conditionals": has_conditionals,
                    "has_parallels": has_parallels,
                    "has_error_handling": has_error_handling,
                    "has_auto_parallel": has_auto_parallel,
                    "depth": depth
                }

            # Fallback for unknown types
            return {
                "type": "Unknown",
                "display": f"[Unknown] {type(comp).__name__}",
                "class_name": type(comp).__name__,
                "depth": depth
            }

        except Exception as e:
            return {
                "type": "Error",
                "display": f"[ERROR] {str(e)[:50]}",
                "error": str(e),
                "depth": depth
            }
        finally:
            visited.discard(comp_id)

    return {"structure": process_component(self)}

print_graph(self)

Enhanced chain visualization with complete functionality coverage and parallel detection.

Source code in toolboxv2/mods/isaa/base/Agent/chain.py

def print_graph(self):
    """Enhanced chain visualization with complete functionality coverage and parallel detection."""

    # Enhanced color scheme with parallel indicators
    COLORS = {
        "Agent": "\033[94m",  # Blue
        "Format": "\033[92m",  # Green
        "Condition": "\033[93m",  # Yellow
        "Parallel": "\033[95m",  # Magenta
        "Function": "\033[35m",  # Light Purple
        "Conditional": "\033[96m",  # Cyan
        "ErrorHandling": "\033[91m",  # Red
        "Chain": "\033[97m",  # White
        "Unknown": "\033[31m",  # Dark Red
        "Error": "\033[91m",  # Red
        "AutoParallel": "\033[105m",  # Bright Magenta Background
    }
    RESET = "\033[0m"
    BOLD = "\033[1m"
    DIM = "\033[2m"
    PARALLEL_ICON = "⚡"
    BRANCH_ICON = "🔀"
    ERROR_ICON = "🚨"
    FUNCTION_ICON = "ƒ"

    def style_component(comp, override_color=None):
        """Apply enhanced styling with parallel indicators."""
        if not comp:
            return f"{COLORS['Error']}[NULL]{RESET}"

        comp_type = comp.get("type", "Unknown")
        display = comp.get("display", f"[{comp_type}]")
        color = override_color or COLORS.get(comp_type, COLORS['Unknown'])
        # Special handling for parallel-creating formats
        if comp_type == "Format" and comp.get("creates_parallel", False):
            return f"{color}{PARALLEL_ICON} {display}{RESET}"
        elif comp_type == "Function":
            return f"{color}{FUNCTION_ICON} {display}{RESET}"
        else:
            color = override_color or COLORS.get(comp_type, COLORS['Unknown'])
            return f"{color}{display}{RESET}"

    def print_section_header(title, details=None):
        """Print formatted section header."""
        print(f"\n{BOLD}{'=' * 60}{RESET}")
        print(f"{BOLD}🔗 {title}{RESET}")
        if details:
            print(f"{DIM}{details}{RESET}")
        print(f"{BOLD}{'=' * 60}{RESET}")

    def render_task_flow(tasks, indent="", show_parallel_creation=True):
        """Render tasks with parallel creation detection."""
        if not tasks:
            print(f"{indent}{DIM}(No tasks){RESET}")
            return

        for i, task in enumerate(tasks):
            if not task:
                continue

            is_last = i == len(tasks) - 1
            connector = "└─ " if is_last else "├─ "
            next_indent = indent + ("    " if is_last else "│   ")

            task_type = task.get("type", "Unknown")

            # Handle different task types
            if task_type == "Format" and task.get("creates_parallel", False):
                print(f"{indent}{connector}{style_component(task)}")

                # Show what happens next
                if i + 1 < len(tasks):
                    next_task = tasks[i + 1]
                    print(f"{next_indent}├─ {DIM}Creates parallel execution for:{RESET}")
                    print(f"{next_indent}└─ {PARALLEL_ICON} {style_component(next_task)}")
                    # Skip the next task in main loop since we showed it here
                    continue

            elif task_type == "Parallel":
                print(f"{indent}{connector}{style_component(task)}")
                branches = task.get("branches", [])

                for j, branch in enumerate(branches):
                    if branch:
                        branch_last = j == len(branches) - 1
                        branch_conn = "└─ " if branch_last else "├─ "
                        branch_indent = next_indent + ("    " if branch_last else "│   ")

                        print(f"{next_indent}{branch_conn}{BRANCH_ICON} Branch {j + 1}:")

                        if branch.get("type") == "Chain":
                            render_task_flow(branch.get("tasks", []), branch_indent, False)
                        else:
                            print(f"{branch_indent}└─ {style_component(branch)}")

            elif task_type == "Conditional":
                print(f"{indent}{connector}{style_component(task)}")

                # Condition
                condition = task.get("condition")
                if condition:
                    print(f"{next_indent}├─ {style_component(condition)}")

                # True branch
                true_branch = task.get("true_branch")
                false_branch = task.get("false_branch")
                has_false = false_branch is not None

                if true_branch:
                    true_conn = "├─ " if has_false else "└─ "
                    print(f"{next_indent}{true_conn}{COLORS['Conditional']}✓ TRUE:{RESET}")
                    true_indent = next_indent + ("│   " if has_false else "    ")

                    if true_branch.get("type") == "Chain":
                        render_task_flow(true_branch.get("tasks", []), true_indent, False)
                    else:
                        print(f"{true_indent}└─ {style_component(true_branch)}")

                if false_branch:
                    print(f"{next_indent}└─ {COLORS['Conditional']}✗ FALSE:{RESET}")
                    false_indent = next_indent + "    "

                    if false_branch.get("type") == "Chain":
                        render_task_flow(false_branch.get("tasks", []), false_indent, False)
                    else:
                        print(f"{false_indent}└─ {style_component(false_branch)}")

            elif task_type == "ErrorHandling":
                print(f"{indent}{connector}{style_component(task)}")

                primary = task.get("primary")
                fallback = task.get("fallback")
                has_fallback = fallback is not None

                if primary:
                    prim_conn = "├─ " if has_fallback else "└─ "
                    print(f"{next_indent}{prim_conn}{COLORS['Chain']}🎯 PRIMARY:{RESET}")
                    prim_indent = next_indent + ("│   " if has_fallback else "    ")

                    if primary.get("type") == "Chain":
                        render_task_flow(primary.get("tasks", []), prim_indent, False)
                    else:
                        print(f"{prim_indent}└─ {style_component(primary)}")

                if fallback:
                    print(f"{next_indent}└─ {ERROR_ICON} FALLBACK:")
                    fallback_indent = next_indent + "    "

                    if fallback.get("type") == "Chain":
                        render_task_flow(fallback.get("tasks", []), fallback_indent, False)
                    else:
                        print(f"{fallback_indent}└─ {style_component(fallback)}")

            else:
                print(f"{indent}{connector}{style_component(task)}")

    # Main execution
    try:
        # Generate graph structure
        graph_data = self.chain_to_graph()
        structure = graph_data.get("structure")

        if not structure:
            print_section_header("Empty Chain")
            return

        # Determine chain characteristics
        chain_type = structure.get("chain_type", "Unknown")
        has_auto_parallel = structure.get("has_auto_parallel", False)
        has_parallels = structure.get("has_parallels", False)
        has_conditionals = structure.get("has_conditionals", False)
        has_error_handling = structure.get("has_error_handling", False)
        task_count = structure.get("task_count", 0)

        # Build header info
        info_parts = [f"Tasks: {task_count}"]
        if has_auto_parallel:
            info_parts.append(f"{PARALLEL_ICON} Auto-Parallel")
        if has_parallels:
            info_parts.append(f"{BRANCH_ICON} Parallel Branches")
        if has_conditionals:
            info_parts.append("🔀 Conditionals")
        if has_error_handling:
            info_parts.append(f"{ERROR_ICON} Error Handling")

        print_section_header(f"Chain Visualization - {chain_type}", " | ".join(info_parts))

        # Handle different structure types
        struct_type = structure.get("type", "Unknown")

        if struct_type == "Chain":
            tasks = structure.get("tasks", [])
            render_task_flow(tasks)

        elif struct_type == "Parallel":
            print(f"{style_component(structure)}")
            branches = structure.get("branches", [])
            for i, branch in enumerate(branches):
                is_last = i == len(branches) - 1
                conn = "└─ " if is_last else "├─ "
                indent = "    " if is_last else "│   "

                print(f"{conn}{BRANCH_ICON} Branch {i + 1}:")
                if branch.get("type") == "Chain":
                    render_task_flow(branch.get("tasks", []), indent, False)
                else:
                    print(f"{indent}└─ {style_component(branch)}")

        elif struct_type == "Conditional" or struct_type == "ErrorHandling":
            render_task_flow([structure])

        else:
            print(f"└─ {style_component(structure)}")

        print(f"\n{DIM}{'─' * 60}{RESET}")

    except Exception as e:
        print(f"\n{COLORS['Error']}{BOLD}[VISUALIZATION ERROR]{RESET}")
        print(f"{COLORS['Error']}Error: {str(e)}{RESET}")

        # Emergency fallback
        print(f"\n{DIM}--- Emergency Info ---{RESET}")
        try:
            attrs = []
            for attr in ['tasks', 'agents', 'condition', 'true_branch', 'false_branch', 'primary', 'fallback']:
                if hasattr(self, attr):
                    val = getattr(self, attr)
                    if val is not None:
                        if isinstance(val, list | tuple):
                            attrs.append(f"{attr}: {len(val)} items")
                        else:
                            attrs.append(f"{attr}: {type(val).__name__}")

            if attrs:
                print("Chain attributes:")
                for attr in attrs:
                    print(f"  • {attr}")
        except:
            print("Complete inspection failed")

        print(f"{DIM}--- End Emergency Info ---{RESET}\n")

config

A2AConfig

Bases: BaseModel

Configuration for A2A integration.

Source code in toolboxv2/mods/isaa/base/Agent/config.py

class A2AConfig(BaseModel):
    """Configuration for A2A integration."""
    server: dict[str, Any] | None = Field(default=None, description="Configuration to run an A2A server (host, port, etc.).")
    known_agents: dict[str, str] = Field(default_factory=dict, description="Named A2A agent URLs to interact with (e.g., {'weather_agent': 'http://weather:5000'}).")
    default_task_timeout: int = Field(default=120, description="Default timeout in seconds for waiting on A2A task results.")

    model_config = ConfigDict(arbitrary_types_allowed=True)

ADKConfig

Bases: BaseModel

Configuration for ADK integration.

Source code in toolboxv2/mods/isaa/base/Agent/config.py

class ADKConfig(BaseModel):
    """Configuration for ADK integration."""
    enabled: bool = Field(default=True, description="Enable ADK features if ADK is installed.")
    description: str | None = Field(default=None, description="ADK LlmAgent description.")
    instruction_override: str | None = Field(default=None, description="Override agent's system message for ADK.")
    # Tools added via builder or auto-discovery
    code_executor: str | BaseCodeExecutor | None = Field(default=None, description="Reference name or instance of ADK code executor.")
    planner: str | BasePlanner | None = Field(default=None, description="Reference name or instance of ADK planner.")
    examples: list[Example] | None = Field(default=None, description="Few-shot examples for ADK.")
    output_schema: type[BaseModel] | None = Field(default=None, description="Pydantic model for structured output.")
    # MCP Toolset config handled separately if ADK is enabled
    use_mcp_toolset: bool = Field(default=True, description="Use ADK's MCPToolset for MCP client connections if ADK is enabled.")
    # Runner config handled separately

    model_config = ConfigDict(arbitrary_types_allowed=True)

AgentConfig

Bases: BaseModel

Main configuration schema for an EnhancedAgent.

Source code in toolboxv2/mods/isaa/base/Agent/config.py

class AgentConfig(BaseModel):
    """Main configuration schema for an EnhancedAgent."""
    agent_name: str = Field(..., description="Unique name for this agent instance.")
    version: str = Field(default="0.1.0")

    agent_instruction: str = Field(default="You are a helpful AI assistant. Answer user questions to the best of your knowledge. Respond concisely. use tools when needed")
    agent_description: str = Field(default="An configurable, production-ready agent with integrated capabilities.")

    # Model Selection
    models: list[ModelConfig] = Field(..., description="List of available LLM configurations.")
    default_llm_model: str = Field(..., description="Name of the ModelConfig to use for general LLM calls.")
    formatter_llm_model: str | None = Field(default=None, description="Optional: Name of a faster/cheaper ModelConfig for a_format_class calls.")

    # Core Agent Settings
    world_model_initial_data: dict[str, Any] | None = Field(default=None)
    enable_streaming: bool = Field(default=False)
    verbose: bool = Field(default=False)
    log_level: str = Field(default="INFO", description="Logging level (DEBUG, INFO, WARNING, ERROR).")
    max_history_length: int = Field(default=20, description="Max conversation turns for LiteLLM history.")
    trim_strategy: Literal["litellm", "basic"] = Field(default="litellm")
    persist_history: bool = Field(default=True, description="Persist conversation history (requires persistent ChatSession).")
    user_id_default: str | None = Field(default=None, description="Default user ID for interactions.")

    # Secure Code Execution
    code_executor_type: Literal["restricted", "docker", "none"] = Field(default="restricted", description="Type of code executor to use.")
    code_executor_config: dict[str, Any] = Field(default_factory=dict, description="Configuration specific to the chosen code executor.")
    enable_adk_code_execution_tool: bool = Field(default=True, description="Expose code execution as an ADK tool if ADK is enabled.")

    # Framework Integrations
    adk: ADKConfig | None = Field(default_factory=ADKConfig if ADK_AVAILABLE_CONF else lambda: None)
    mcp: MCPConfig | None = Field(default_factory=MCPConfig if MCP_AVAILABLE_CONF else lambda: None)
    a2a: A2AConfig | None = Field(default_factory=A2AConfig if A2A_AVAILABLE_CONF else lambda: None)

    # Observability & Cost
    observability: ObservabilityConfig | None = Field(default_factory=ObservabilityConfig)
    budget_manager: BudgetManager | None = Field(default=None, description="Global LiteLLM budget manager instance.") # Needs to be passed in

    # Human-in-the-Loop
    enable_hitl: bool = Field(default=False, description="Enable basic Human-in-the-Loop hooks.")

    # Add other global settings as needed

    model_config = ConfigDict(arbitrary_types_allowed=True)

    @model_validator(mode='after')
    def validate_model_references(self) -> 'AgentConfig':
        model_names = {m.name for m in self.models}
        if self.default_llm_model not in model_names:
            raise ValueError(f"default_llm_model '{self.default_llm_model}' not found in defined models.")
        if self.formatter_llm_model and self.formatter_llm_model not in model_names:
            raise ValueError(f"formatter_llm_model '{self.formatter_llm_model}' not found in defined models.")
        return self

    @model_validator(mode='after')
    def validate_framework_availability(self) -> 'AgentConfig':
        if self.adk and self.adk.enabled and not ADK_AVAILABLE_CONF:
            logger.warning("ADK configuration provided but ADK library not installed. Disabling ADK features.")
            self.adk.enabled = False
        if self.mcp and (self.mcp.server or self.mcp.client_connections) and not MCP_AVAILABLE_CONF:
             logger.warning("MCP configuration provided but MCP library not installed. Disabling MCP features.")
             self.mcp = None # Or disable specific parts
        if self.a2a and (self.a2a.server or self.a2a.known_agents) and not A2A_AVAILABLE_CONF:
             logger.warning("A2A configuration provided but A2A library not installed. Disabling A2A features.")
             self.a2a = None # Or disable specific parts
        return self

    @classmethod
    def load_from_yaml(cls, path: str | Path) -> 'AgentConfig':
        """Loads configuration from a YAML file."""
        file_path = Path(path)
        if not file_path.is_file():
            raise FileNotFoundError(f"Configuration file not found: {path}")
        with open(file_path) as f:
            config_data = yaml.safe_load(f)
        logger.info(f"Loaded agent configuration from {path}")
        return cls(**config_data)

    def save_to_yaml(self, path: str | Path):
        """Saves the current configuration to a YAML file."""
        file_path = Path(path)
        file_path.parent.mkdir(parents=True, exist_ok=True)
        with open(file_path, 'w') as f:
            # Use Pydantic's model_dump for clean serialization
            yaml.dump(self.model_dump(mode='python'), f, sort_keys=False)
        logger.info(f"Saved agent configuration to {path}")

load_from_yaml(path) classmethod

Loads configuration from a YAML file.

Source code in toolboxv2/mods/isaa/base/Agent/config.py

@classmethod
def load_from_yaml(cls, path: str | Path) -> 'AgentConfig':
    """Loads configuration from a YAML file."""
    file_path = Path(path)
    if not file_path.is_file():
        raise FileNotFoundError(f"Configuration file not found: {path}")
    with open(file_path) as f:
        config_data = yaml.safe_load(f)
    logger.info(f"Loaded agent configuration from {path}")
    return cls(**config_data)

save_to_yaml(path)

Saves the current configuration to a YAML file.

Source code in toolboxv2/mods/isaa/base/Agent/config.py

def save_to_yaml(self, path: str | Path):
    """Saves the current configuration to a YAML file."""
    file_path = Path(path)
    file_path.parent.mkdir(parents=True, exist_ok=True)
    with open(file_path, 'w') as f:
        # Use Pydantic's model_dump for clean serialization
        yaml.dump(self.model_dump(mode='python'), f, sort_keys=False)
    logger.info(f"Saved agent configuration to {path}")

MCPConfig

Bases: BaseModel

Configuration for MCP integration.

Source code in toolboxv2/mods/isaa/base/Agent/config.py

class MCPConfig(BaseModel):
    """Configuration for MCP integration."""
    server: dict[str, Any] | None = Field(default=None, description="Configuration to run an MCP server (host, port, etc.).")
    client_connections: dict[str, str] = Field(default_factory=dict, description="Named MCP server URLs to connect to as a client (e.g., {'files': 'stdio:npx @mcp/server-filesystem /data'}).")
    # ADK's MCPToolset handles client connections if ADKConfig.use_mcp_toolset is True

    model_config = ConfigDict(arbitrary_types_allowed=True)

ModelConfig

Bases: BaseModel

Configuration specific to an LLM model via LiteLLM.

Source code in toolboxv2/mods/isaa/base/Agent/config.py

class ModelConfig(BaseModel):
    """Configuration specific to an LLM model via LiteLLM."""
    # Used as key for model selection
    name: str = Field(..., description="Unique identifier/alias for this model configuration (e.g., 'fast_formatter', 'main_reasoner').")
    model: str = Field(..., description="LiteLLM model string (e.g., 'gemini/gemini-1.5-pro-latest', 'ollama/mistral').")
    provider: str | None = Field(default=None, description="LiteLLM provider override if needed.")
    api_key: str | None = Field(default=None, description="API Key (consider using environment variables).")
    api_base: str | None = Field(default=None, description="API Base URL (for local models, proxies).")
    api_version: str | None = Field(default=None, description="API Version (e.g., for Azure).")

    # Common LLM Parameters
    temperature: float | None = Field(default=0.7)
    top_p: float | None = Field(default=None)
    top_k: int | None = Field(default=None)
    max_tokens: int | None = Field(default=2048, description="Max tokens for generation.")
    max_input_tokens: int | None = Field(default=None, description="Max input context window (autodetected if None).")
    stop_sequence: list[str] | None = Field(default=None)
    presence_penalty: float | None = Field(default=None)
    frequency_penalty: float | None = Field(default=None)
    system_message: str | None = Field(default=None, description="Default system message for this model.")

    # LiteLLM Specific
    caching: bool = Field(default=True, description="Enable LiteLLM caching for this model.")
    # budget_manager: Optional[BudgetManager] = Field(default=None) # Budget manager applied globally or per-agent

    model_config = ConfigDict(arbitrary_types_allowed=True, extra='allow') # Allow extra LiteLLM params

ObservabilityConfig

Bases: BaseModel

Configuration for observability (OpenTelemetry).

Source code in toolboxv2/mods/isaa/base/Agent/config.py

class ObservabilityConfig(BaseModel):
    """Configuration for observability (OpenTelemetry)."""
    enabled: bool = Field(default=True)
    endpoint: str | None = Field(default=None, description="OTLP endpoint URL (e.g., http://jaeger:4317).")
    service_name: str | None = Field(default=None, description="Service name for traces/metrics (defaults to agent name).")
    # Add more OTel config options as needed (headers, certs, resource attributes)

    model_config = ConfigDict(arbitrary_types_allowed=True)

executors

DockerCodeExecutor

Bases: _BaseExecutorClass

Executes Python code in a sandboxed Docker container.

Requires Docker to be installed and running, and the 'docker' Python SDK.

Source code in toolboxv2/mods/isaa/base/Agent/executors.py

class DockerCodeExecutor(_BaseExecutorClass):
    """
    Executes Python code in a sandboxed Docker container.

    Requires Docker to be installed and running, and the 'docker' Python SDK.
    """
    DEFAULT_DOCKER_IMAGE = "python:3.10-slim" # Use a minimal image
    DEFAULT_TIMEOUT = 10 # Seconds
    DEFAULT_MEM_LIMIT = "128m"
    DEFAULT_CPUS = 0.5

    def __init__(self,
                 docker_image: str = DEFAULT_DOCKER_IMAGE,
                 timeout: int = DEFAULT_TIMEOUT,
                 mem_limit: str = DEFAULT_MEM_LIMIT,
                 cpus: float = DEFAULT_CPUS,
                 network_mode: str = "none", # Disable networking by default for security
                 docker_client_config: dict | None = None):
        if not DOCKER_AVAILABLE:
            raise ImportError("Docker SDK not installed ('pip install docker'). Cannot use DockerCodeExecutor.")

        self.docker_image = docker_image
        self.timeout = timeout
        self.mem_limit = mem_limit
        self.cpus = cpus
        self.network_mode = network_mode
        try:
            self.client = docker.from_env(**(docker_client_config or {}))
            self.client.ping() # Check connection
            # Ensure image exists locally or pull it
            try:
                self.client.images.get(self.docker_image)
                logger.info(f"Docker image '{self.docker_image}' found locally.")
            except ImageNotFound:
                logger.warning(f"Docker image '{self.docker_image}' not found locally. Attempting to pull...")
                try:
                    self.client.images.pull(self.docker_image)
                    logger.info(f"Successfully pulled Docker image '{self.docker_image}'.")
                except APIError as pull_err:
                    raise RuntimeError(f"Failed to pull Docker image '{self.docker_image}': {pull_err}") from pull_err
        except Exception as e:
            raise RuntimeError(f"Failed to connect to Docker daemon: {e}. Is Docker running?") from e
        logger.info(f"DockerCodeExecutor initialized (Image: {docker_image}, Timeout: {timeout}s, Network: {network_mode})")

    def _execute(self, code: str) -> dict[str, Any]:
        """Internal execution logic."""
        result = {"stdout": "", "stderr": "", "error": None, "exit_code": None}
        container = None

        try:
            logger.debug(f"Creating Docker container from image '{self.docker_image}'...")
            container = self.client.containers.run(
                image=self.docker_image,
                command=["python", "-c", code],
                detach=True,
                mem_limit=self.mem_limit,
                nano_cpus=int(self.cpus * 1e9),
                network_mode=self.network_mode,
                # Security considerations: Consider read-only filesystem, dropping capabilities
                read_only=True,
                # working_dir="/app", # Define a working dir if needed
                # volumes={...} # Mount volumes carefully if required
            )
            logger.debug(f"Container '{container.short_id}' started.")

            # Wait for container completion with timeout
            container_result = container.wait(timeout=self.timeout)
            result["exit_code"] = container_result.get("StatusCode", None)

            # Retrieve logs
            result["stdout"] = container.logs(stdout=True, stderr=False).decode('utf-8', errors='replace').strip()
            result["stderr"] = container.logs(stdout=False, stderr=True).decode('utf-8', errors='replace').strip()

            logger.debug(f"Container '{container.short_id}' finished with exit code {result['exit_code']}.")
            if result["exit_code"] != 0:
                 logger.warning(f"Container stderr: {result['stderr'][:500]}...") # Log stderr on failure

        except ContainerError as e:
            result["error"] = f"ContainerError: {e}"
            result["stderr"] = e.stderr.decode('utf-8', errors='replace').strip() if e.stderr else str(e)
            result["exit_code"] = e.exit_status
            logger.error(f"Container '{container.short_id if container else 'N/A'}' failed: {result['error']}\nStderr: {result['stderr']}")
        except APIError as e:
            result["error"] = f"Docker APIError: {e}"
            result["exit_code"] = -1
            logger.error(f"Docker API error during execution: {e}")
        except Exception as e:
            # Catch potential timeout errors from container.wait or other unexpected issues
            result["error"] = f"Unexpected execution error: {type(e).__name__}: {e}"
            result["exit_code"] = -1
            # Check if it looks like a timeout
            if isinstance(e, TimeoutError) or "Timeout" in str(e): # docker SDK might raise requests.exceptions.ReadTimeout
                result["stderr"] = f"Execution timed out after {self.timeout} seconds."
                logger.warning(f"Container execution timed out ({self.timeout}s).")
            else:
                logger.error(f"Unexpected error during Docker execution: {e}", exc_info=True)
        finally:
            if container:
                try:
                    logger.debug(f"Removing container '{container.short_id}'...")
                    container.remove(force=True)
                except APIError as rm_err:
                    logger.warning(f"Failed to remove container {container.short_id}: {rm_err}")

        return result

     # --- ADK Compatibility Method ---
    if ADK_EXEC_AVAILABLE:
        def execute_code(self, invocation_context: InvocationContext, code_input: CodeExecutionInput) -> CodeExecutionResult:
            logger.debug(f"DockerCodeExecutor executing ADK request (lang: {code_input.language}). Code: {code_input.code[:100]}...")
            if code_input.language.lower() != 'python':
                 return CodeExecutionResult(output=f"Error: Unsupported language '{code_input.language}'. Only Python is supported.", outcome="OUTCOME_FAILURE")

            exec_result = self._execute(code_input.code)

            output_str = ""
            if exec_result["stdout"]:
                output_str += f"Stdout:\n{exec_result['stdout']}\n"
            if exec_result["stderr"]:
                 output_str += f"Stderr:\n{exec_result['stderr']}\n"
            if not output_str and exec_result["exit_code"] == 0:
                 output_str = "Execution successful with no output."
            elif not output_str and exec_result["exit_code"] != 0:
                 output_str = f"Execution failed with no output (Exit code: {exec_result['exit_code']}). Error: {exec_result['error']}"

            outcome = "OUTCOME_SUCCESS" if exec_result["exit_code"] == 0 else "OUTCOME_FAILURE"

            return CodeExecutionResult(output=output_str.strip(), outcome=outcome)
    # --- End ADK Compatibility ---

    # --- Direct Call Method ---
    def execute(self, code: str) -> dict[str, Any]:
        """Directly execute code, returning detailed dictionary."""
        logger.debug(f"DockerCodeExecutor executing direct call. Code: {code[:100]}...")
        return self._execute(code)

execute(code)

Directly execute code, returning detailed dictionary.

Source code in toolboxv2/mods/isaa/base/Agent/executors.py

def execute(self, code: str) -> dict[str, Any]:
    """Directly execute code, returning detailed dictionary."""
    logger.debug(f"DockerCodeExecutor executing direct call. Code: {code[:100]}...")
    return self._execute(code)

RestrictedPythonExecutor

Bases: _BaseExecutorClass

Executes Python code using restrictedpython.

Safer than exec() but NOT a full sandbox. Known vulnerabilities exist. Use with extreme caution and only with trusted code sources or for low-risk operations. Docker is strongly recommended for untrusted code.

Source code in toolboxv2/mods/isaa/base/Agent/executors.py

class RestrictedPythonExecutor(_BaseExecutorClass):
    """
    Executes Python code using restrictedpython.

    Safer than exec() but NOT a full sandbox. Known vulnerabilities exist.
    Use with extreme caution and only with trusted code sources or for
    low-risk operations. Docker is strongly recommended for untrusted code.
    """
    DEFAULT_ALLOWED_GLOBALS = {
        **safe_globals,
        '_print_': restrictedpython.PrintCollector,
        '_getattr_': restrictedpython.safe_getattr,
        '_getitem_': restrictedpython.safe_getitem,
        '_write_': restrictedpython.guarded_setattr, # Allows modifying specific safe objects if needed
        # Add other safe builtins or modules carefully
        'math': __import__('math'),
        'random': __import__('random'),
        'datetime': __import__('datetime'),
        'time': __import__('time'),
        # 'requests': None, # Example: Explicitly disallow
    }

    def __init__(self, allowed_globals: dict | None = None, max_execution_time: int = 5):
        if not RESTRICTEDPYTHON_AVAILABLE:
            raise ImportError("restrictedpython is not installed. Cannot use RestrictedPythonExecutor.")
        self.allowed_globals = allowed_globals or self.DEFAULT_ALLOWED_GLOBALS
        self.max_execution_time = max_execution_time # Basic timeout (not perfectly enforced by restrictedpython)
        logger.warning("Initialized RestrictedPythonExecutor. This provides LIMITED sandboxing. Use Docker for untrusted code.")

    def _execute(self, code: str) -> dict[str, Any]:
        """Internal execution logic."""
        start_time = time.monotonic()
        result = {"stdout": "", "stderr": "", "error": None, "exit_code": None}
        local_vars = {}
        stdout_capture = io.StringIO()
        stderr_capture = io.StringIO()

        try:
            # Basic timeout check (not preemptive)
            if time.monotonic() - start_time > self.max_execution_time:
                 raise TimeoutError(f"Execution exceeded max time of {self.max_execution_time}s (pre-check).")

            # Compile the code in restricted mode
            byte_code = compile_restricted(code, filename='<inline code>', mode='exec')

            # Add a print collector to capture output
            self.allowed_globals['_print_'] = restrictedpython.PrintCollector
            print_collector = self.allowed_globals['_print_']()
            exec_globals = {**self.allowed_globals, '_print': print_collector}

            # Execute the compiled code
            # Note: restrictedpython does not inherently support robust timeouts during exec
            exec(byte_code, exec_globals, local_vars)

            # Check execution time again
            duration = time.monotonic() - start_time
            if duration > self.max_execution_time:
                logger.warning(f"Execution finished but exceeded max time ({duration:.2f}s > {self.max_execution_time}s).")
                # Potentially treat as an error or partial success

            result["stdout"] = print_collector.printed_text # Access collected prints
            result["exit_code"] = 0 # Assume success if no exception

        except TimeoutError as e:
            result["stderr"] = f"TimeoutError: {e}"
            result["error"] = str(e)
            result["exit_code"] = -1 # Indicate timeout
        except SyntaxError as e:
            result["stderr"] = f"SyntaxError: {e}"
            result["error"] = str(e)
            result["exit_code"] = 1
        except Exception as e:
            # Capture other potential execution errors allowed by restrictedpython
            error_type = type(e).__name__
            error_msg = f"{error_type}: {e}"
            result["stderr"] = error_msg
            result["error"] = str(e)
            result["exit_code"] = 1
            logger.warning(f"RestrictedPython execution caught exception: {error_msg}", exc_info=False) # Avoid logging potentially sensitive details from code
        finally:
            stdout_capture.close() # Not used directly with PrintCollector
            stderr_capture.close()

        return result

    # --- ADK Compatibility Method ---
    if ADK_EXEC_AVAILABLE:
        def execute_code(self, invocation_context: InvocationContext, code_input: CodeExecutionInput) -> CodeExecutionResult:
            logger.debug(f"RestrictedPythonExecutor executing ADK request (lang: {code_input.language}). Code: {code_input.code[:100]}...")
            if code_input.language.lower() != 'python':
                 return CodeExecutionResult(output=f"Error: Unsupported language '{code_input.language}'. Only Python is supported.", outcome="OUTCOME_FAILURE")

            exec_result = self._execute(code_input.code)

            output_str = ""
            if exec_result["stdout"]:
                output_str += f"Stdout:\n{exec_result['stdout']}\n"
            if exec_result["stderr"]:
                 output_str += f"Stderr:\n{exec_result['stderr']}\n"
            if not output_str and exec_result["exit_code"] == 0:
                 output_str = "Execution successful with no output."
            elif not output_str and exec_result["exit_code"] != 0:
                 output_str = f"Execution failed with no output (Exit code: {exec_result['exit_code']}). Error: {exec_result['error']}"


            outcome = "OUTCOME_SUCCESS" if exec_result["exit_code"] == 0 else "OUTCOME_FAILURE"

            return CodeExecutionResult(output=output_str.strip(), outcome=outcome)
    # --- End ADK Compatibility ---

    # --- Direct Call Method ---
    def execute(self, code: str) -> dict[str, Any]:
        """Directly execute code, returning detailed dictionary."""
        logger.debug(f"RestrictedPythonExecutor executing direct call. Code: {code[:100]}...")
        return self._execute(code)

execute(code)

Directly execute code, returning detailed dictionary.

Source code in toolboxv2/mods/isaa/base/Agent/executors.py

def execute(self, code: str) -> dict[str, Any]:
    """Directly execute code, returning detailed dictionary."""
    logger.debug(f"RestrictedPythonExecutor executing direct call. Code: {code[:100]}...")
    return self._execute(code)

get_code_executor(config)

Creates a code executor instance based on configuration.

Source code in toolboxv2/mods/isaa/base/Agent/executors.py

def get_code_executor(config: 'AgentConfig') -> RestrictedPythonExecutor | DockerCodeExecutor | BaseCodeExecutor | None:
    """Creates a code executor instance based on configuration."""
    executor_type = config.code_executor_type
    executor_config = config.code_executor_config or {}

    if executor_type == "restricted":
        if not RESTRICTEDPYTHON_AVAILABLE:
            logger.error("RestrictedPython executor configured but library not installed. Code execution disabled.")
            return None
        return RestrictedPythonExecutor(**executor_config)
    elif executor_type == "docker":
        if not DOCKER_AVAILABLE:
            logger.error("Docker executor configured but library not installed or Docker not running. Code execution disabled.")
            return None
        try:
            return DockerCodeExecutor(**executor_config)
        except Exception as e:
            logger.error(f"Failed to initialize DockerCodeExecutor: {e}. Code execution disabled.")
            return None
    elif executor_type == "none":
        logger.info("Code execution explicitly disabled in configuration.")
        return None
    elif executor_type and ADK_EXEC_AVAILABLE and isinstance(executor_type, BaseCodeExecutor):
        # Allow passing a pre-configured ADK executor instance
        logger.info(f"Using pre-configured ADK code executor: {type(executor_type).__name__}")
        return executor_type
    else:
        logger.warning(f"Unknown or unsupported code_executor_type: '{executor_type}'. Code execution disabled.")
        return None

types

AgentCheckpoint dataclass

Enhanced AgentCheckpoint with UnifiedContextManager and ChatSession integration

Source code in toolboxv2/mods/isaa/base/Agent/types.py

@dataclass
class AgentCheckpoint:
    """Enhanced AgentCheckpoint with UnifiedContextManager and ChatSession integration"""
    timestamp: datetime
    agent_state: dict[str, Any]
    task_state: dict[str, Any]
    world_model: dict[str, Any]
    active_flows: list[str]
    metadata: dict[str, Any] = field(default_factory=dict)

    # NEUE: Enhanced checkpoint data for UnifiedContextManager integration
    session_data: dict[str, Any] = field(default_factory=dict)
    context_manager_state: dict[str, Any] = field(default_factory=dict)
    conversation_history: list[dict[str, Any]] = field(default_factory=list)
    variable_system_state: dict[str, Any] = field(default_factory=dict)
    results_store: dict[str, Any] = field(default_factory=dict)
    tool_capabilities: dict[str, Any] = field(default_factory=dict)
    variable_scopes: dict[str, Any] = field(default_factory=dict)

    # Optional: Additional system state
    performance_metrics: dict[str, Any] = field(default_factory=dict)
    execution_history: list[dict[str, Any]] = field(default_factory=list)

    def get_checkpoint_summary(self) -> str:
        """Get human-readable checkpoint summary"""
        try:
            summary_parts = []

            # Basic info
            if self.session_data:
                session_count = len([s for s in self.session_data.values() if s.get("status") != "failed"])
                summary_parts.append(f"{session_count} sessions")

            # Task info
            if self.task_state:
                completed_tasks = len([t for t in self.task_state.values() if t.get("status") == "completed"])
                total_tasks = len(self.task_state)
                summary_parts.append(f"{completed_tasks}/{total_tasks} tasks")

            # Conversation info
            if self.conversation_history:
                summary_parts.append(f"{len(self.conversation_history)} messages")

            # Context info
            if self.context_manager_state:
                cache_count = self.context_manager_state.get("cache_entries", 0)
                if cache_count > 0:
                    summary_parts.append(f"{cache_count} cached contexts")

            # Variable system info
            if self.variable_system_state:
                scopes = len(self.variable_system_state.get("scopes", {}))
                summary_parts.append(f"{scopes} variable scopes")

            # Tool capabilities
            if self.tool_capabilities:
                summary_parts.append(f"{len(self.tool_capabilities)} analyzed tools")

            return "; ".join(summary_parts) if summary_parts else "Basic checkpoint"

        except Exception as e:
            return f"Summary generation failed: {str(e)}"

    def get_storage_size_estimate(self) -> dict[str, int]:
        """Estimate storage size of different checkpoint components"""
        try:
            sizes = {}

            # Calculate sizes in bytes (approximate)
            sizes["agent_state"] = len(str(self.agent_state))
            sizes["task_state"] = len(str(self.task_state))
            sizes["world_model"] = len(str(self.world_model))
            sizes["conversation_history"] = len(str(self.conversation_history))
            sizes["session_data"] = len(str(self.session_data))
            sizes["context_manager_state"] = len(str(self.context_manager_state))
            sizes["variable_system_state"] = len(str(self.variable_system_state))
            sizes["results_store"] = len(str(self.results_store))
            sizes["tool_capabilities"] = len(str(self.tool_capabilities))

            sizes["total_bytes"] = sum(sizes.values())
            sizes["total_kb"] = sizes["total_bytes"] / 1024
            sizes["total_mb"] = sizes["total_kb"] / 1024

            return sizes

        except Exception as e:
            return {"error": str(e)}

    def validate_checkpoint_integrity(self) -> dict[str, Any]:
        """Validate checkpoint integrity and completeness"""
        validation = {
            "is_valid": True,
            "errors": [],
            "warnings": [],
            "completeness_score": 0.0,
            "components_present": []
        }

        try:
            # Check required components
            required_components = ["timestamp", "agent_state", "task_state", "world_model", "active_flows"]
            for component in required_components:
                if hasattr(self, component) and getattr(self, component) is not None:
                    validation["components_present"].append(component)
                else:
                    validation["errors"].append(f"Missing required component: {component}")
                    validation["is_valid"] = False

            # Check optional enhanced components
            enhanced_components = ["session_data", "context_manager_state", "conversation_history",
                                   "variable_system_state", "results_store", "tool_capabilities"]

            for component in enhanced_components:
                if hasattr(self, component) and getattr(self, component):
                    validation["components_present"].append(component)

            # Calculate completeness score
            total_possible = len(required_components) + len(enhanced_components)
            validation["completeness_score"] = len(validation["components_present"]) / total_possible

            # Check timestamp validity
            if isinstance(self.timestamp, datetime):
                age_hours = (datetime.now() - self.timestamp).total_seconds() / 3600
                if age_hours > 24:
                    validation["warnings"].append(f"Checkpoint is {age_hours:.1f} hours old")
            else:
                validation["errors"].append("Invalid timestamp format")
                validation["is_valid"] = False

            # Check session data consistency
            if self.session_data and self.conversation_history:
                session_ids_in_data = set(self.session_data.keys())
                session_ids_in_conversation = set(
                    msg.get("session_id") for msg in self.conversation_history
                    if msg.get("session_id")
                )

                if session_ids_in_data != session_ids_in_conversation:
                    validation["warnings"].append("Session data and conversation history session IDs don't match")

            return validation

        except Exception as e:
            validation["errors"].append(f"Validation error: {str(e)}")
            validation["is_valid"] = False
            return validation

    def get_version_info(self) -> dict[str, str]:
        """Get checkpoint version information"""
        return {
            "checkpoint_version": self.metadata.get("checkpoint_version", "1.0"),
            "data_format": "enhanced" if self.session_data or self.context_manager_state else "basic",
            "context_system": "unified" if self.context_manager_state else "legacy",
            "variable_system": "integrated" if self.variable_system_state else "basic",
            "session_management": "chatsession" if self.session_data else "memory_only",
            "created_with": "FlowAgent v2.0 Enhanced Context System"
        }

get_checkpoint_summary()

Get human-readable checkpoint summary

Source code in toolboxv2/mods/isaa/base/Agent/types.py

def get_checkpoint_summary(self) -> str:
    """Get human-readable checkpoint summary"""
    try:
        summary_parts = []

        # Basic info
        if self.session_data:
            session_count = len([s for s in self.session_data.values() if s.get("status") != "failed"])
            summary_parts.append(f"{session_count} sessions")

        # Task info
        if self.task_state:
            completed_tasks = len([t for t in self.task_state.values() if t.get("status") == "completed"])
            total_tasks = len(self.task_state)
            summary_parts.append(f"{completed_tasks}/{total_tasks} tasks")

        # Conversation info
        if self.conversation_history:
            summary_parts.append(f"{len(self.conversation_history)} messages")

        # Context info
        if self.context_manager_state:
            cache_count = self.context_manager_state.get("cache_entries", 0)
            if cache_count > 0:
                summary_parts.append(f"{cache_count} cached contexts")

        # Variable system info
        if self.variable_system_state:
            scopes = len(self.variable_system_state.get("scopes", {}))
            summary_parts.append(f"{scopes} variable scopes")

        # Tool capabilities
        if self.tool_capabilities:
            summary_parts.append(f"{len(self.tool_capabilities)} analyzed tools")

        return "; ".join(summary_parts) if summary_parts else "Basic checkpoint"

    except Exception as e:
        return f"Summary generation failed: {str(e)}"

get_storage_size_estimate()

Estimate storage size of different checkpoint components

Source code in toolboxv2/mods/isaa/base/Agent/types.py

def get_storage_size_estimate(self) -> dict[str, int]:
    """Estimate storage size of different checkpoint components"""
    try:
        sizes = {}

        # Calculate sizes in bytes (approximate)
        sizes["agent_state"] = len(str(self.agent_state))
        sizes["task_state"] = len(str(self.task_state))
        sizes["world_model"] = len(str(self.world_model))
        sizes["conversation_history"] = len(str(self.conversation_history))
        sizes["session_data"] = len(str(self.session_data))
        sizes["context_manager_state"] = len(str(self.context_manager_state))
        sizes["variable_system_state"] = len(str(self.variable_system_state))
        sizes["results_store"] = len(str(self.results_store))
        sizes["tool_capabilities"] = len(str(self.tool_capabilities))

        sizes["total_bytes"] = sum(sizes.values())
        sizes["total_kb"] = sizes["total_bytes"] / 1024
        sizes["total_mb"] = sizes["total_kb"] / 1024

        return sizes

    except Exception as e:
        return {"error": str(e)}

get_version_info()

Get checkpoint version information

Source code in toolboxv2/mods/isaa/base/Agent/types.py

def get_version_info(self) -> dict[str, str]:
    """Get checkpoint version information"""
    return {
        "checkpoint_version": self.metadata.get("checkpoint_version", "1.0"),
        "data_format": "enhanced" if self.session_data or self.context_manager_state else "basic",
        "context_system": "unified" if self.context_manager_state else "legacy",
        "variable_system": "integrated" if self.variable_system_state else "basic",
        "session_management": "chatsession" if self.session_data else "memory_only",
        "created_with": "FlowAgent v2.0 Enhanced Context System"
    }

validate_checkpoint_integrity()

Validate checkpoint integrity and completeness

Source code in toolboxv2/mods/isaa/base/Agent/types.py

def validate_checkpoint_integrity(self) -> dict[str, Any]:
    """Validate checkpoint integrity and completeness"""
    validation = {
        "is_valid": True,
        "errors": [],
        "warnings": [],
        "completeness_score": 0.0,
        "components_present": []
    }

    try:
        # Check required components
        required_components = ["timestamp", "agent_state", "task_state", "world_model", "active_flows"]
        for component in required_components:
            if hasattr(self, component) and getattr(self, component) is not None:
                validation["components_present"].append(component)
            else:
                validation["errors"].append(f"Missing required component: {component}")
                validation["is_valid"] = False

        # Check optional enhanced components
        enhanced_components = ["session_data", "context_manager_state", "conversation_history",
                               "variable_system_state", "results_store", "tool_capabilities"]

        for component in enhanced_components:
            if hasattr(self, component) and getattr(self, component):
                validation["components_present"].append(component)

        # Calculate completeness score
        total_possible = len(required_components) + len(enhanced_components)
        validation["completeness_score"] = len(validation["components_present"]) / total_possible

        # Check timestamp validity
        if isinstance(self.timestamp, datetime):
            age_hours = (datetime.now() - self.timestamp).total_seconds() / 3600
            if age_hours > 24:
                validation["warnings"].append(f"Checkpoint is {age_hours:.1f} hours old")
        else:
            validation["errors"].append("Invalid timestamp format")
            validation["is_valid"] = False

        # Check session data consistency
        if self.session_data and self.conversation_history:
            session_ids_in_data = set(self.session_data.keys())
            session_ids_in_conversation = set(
                msg.get("session_id") for msg in self.conversation_history
                if msg.get("session_id")
            )

            if session_ids_in_data != session_ids_in_conversation:
                validation["warnings"].append("Session data and conversation history session IDs don't match")

        return validation

    except Exception as e:
        validation["errors"].append(f"Validation error: {str(e)}")
        validation["is_valid"] = False
        return validation

AgentModelData

Bases: BaseModel

Source code in toolboxv2/mods/isaa/base/Agent/types.py

class AgentModelData(BaseModel):
    name: str = "FlowAgent"
    fast_llm_model: str = "openrouter/anthropic/claude-3-haiku"
    complex_llm_model: str = "openrouter/openai/gpt-4o"
    system_message: str = "You are a production-ready autonomous agent."
    temperature: float = 0.7
    max_tokens: int = 2048
    max_input_tokens: int = 32768
    api_key: str | None  = None
    api_base: str | None  = None
    budget_manager: Any  = None
    caching: bool = True
    persona: PersonaConfig | None = True
    use_fast_response: bool = True

    def get_system_message_with_persona(self) -> str:
        """Get system message with persona integration"""
        base_message = self.system_message

        if self.persona and self.persona.apply_method in ["system_prompt", "both"]:
            persona_addition = self.persona.to_system_prompt_addition()
            if persona_addition:
                base_message += f"\n## Persona Instructions\n{persona_addition}"

        return base_message

get_system_message_with_persona()

Get system message with persona integration

Source code in toolboxv2/mods/isaa/base/Agent/types.py

def get_system_message_with_persona(self) -> str:
    """Get system message with persona integration"""
    base_message = self.system_message

    if self.persona and self.persona.apply_method in ["system_prompt", "both"]:
        persona_addition = self.persona.to_system_prompt_addition()
        if persona_addition:
            base_message += f"\n## Persona Instructions\n{persona_addition}"

    return base_message

ChainMetadata dataclass

Metadata for stored chains

Source code in toolboxv2/mods/isaa/base/Agent/types.py

@dataclass
class ChainMetadata:
    """Metadata for stored chains"""
    name: str
    description: str = ""
    created_at: datetime = field(default_factory=datetime.now)
    modified_at: datetime = field(default_factory=datetime.now)
    version: str = "1.0.0"
    tags: list[str] = field(default_factory=list)
    author: str = ""
    complexity: str = "simple"  # simple, medium, complex
    agent_count: int = 0
    has_conditionals: bool = False
    has_parallels: bool = False
    has_error_handling: bool = False

DecisionTask dataclass

Bases: Task

Task für dynamisches Routing

Source code in toolboxv2/mods/isaa/base/Agent/types.py

@dataclass
class DecisionTask(Task):
    """Task für dynamisches Routing"""
    decision_prompt: str = ""  # Kurze Frage an LLM
    routing_map: dict[str, str] = field(default_factory=dict)  # Ergebnis -> nächster Task
    decision_model: str = "fast"  # Welches LLM für Entscheidung

FormatConfig dataclass

Konfiguration für Response-Format und -Länge

Source code in toolboxv2/mods/isaa/base/Agent/types.py

@dataclass
class FormatConfig:
    """Konfiguration für Response-Format und -Länge"""
    response_format: ResponseFormat = ResponseFormat.FREE_TEXT
    text_length: TextLength = TextLength.CHAT_CONVERSATION
    custom_instructions: str = ""
    strict_format_adherence: bool = True
    quality_threshold: float = 0.7

    def get_format_instructions(self) -> str:
        """Generiere Format-spezifische Anweisungen"""
        format_instructions = {
            ResponseFormat.FREE_TEXT: "Use natural continuous text without special formatting.",
            ResponseFormat.WITH_TABLES: "Integrate tables for structured data representation. Use Markdown tables.",
            ResponseFormat.WITH_BULLET_POINTS: "Structure information with bullet points (•, -, *) for better readability.",
            ResponseFormat.WITH_LISTS: "Use numbered and unnumbered lists to organize content.",
            ResponseFormat.TEXT_ONLY: "Plain text only without formatting, symbols, or structural elements.",
            ResponseFormat.MD_TEXT: "Full Markdown formatting with headings, code blocks, links, etc.",
            ResponseFormat.YAML_TEXT: "Structure responses in YAML format for machine-readable output.",
            ResponseFormat.JSON_TEXT: "Format responses as a JSON structure for API integration.",
            ResponseFormat.PSEUDO_CODE: "Use pseudocode structure for algorithmic or logical explanations.",
            ResponseFormat.CODE_STRUCTURE: "Structure like code with indentation, comments, and logical blocks."
        }
        return format_instructions.get(self.response_format, "Standard-Formatierung.")

    def get_length_instructions(self) -> str:
        """Generiere Längen-spezifische Anweisungen"""
        length_instructions = {
            TextLength.MINI_CHAT: "Very short, concise answers (1–2 sentences, max 50 words). Chat style.",
            TextLength.CHAT_CONVERSATION: "Moderate conversation length (2–4 sentences, 50–150 words). Natural conversational style.",
            TextLength.TABLE_CONVERSATION: "Structured, tabular presentation with compact explanations (100–250 words).",
            TextLength.DETAILED_INDEPTH: "Comprehensive, detailed explanations (300–800 words) with depth and context.",
            TextLength.PHD_LEVEL: "Academic depth with extensive explanations (800+ words), references, and technical terminology."
        }
        return length_instructions.get(self.text_length, "Standard-Länge.")

    def get_combined_instructions(self) -> str:
        """Kombiniere Format- und Längen-Anweisungen"""
        instructions = []
        instructions.append("## Format-Anforderungen:")
        instructions.append(self.get_format_instructions())
        instructions.append("\n## Längen-Anforderungen:")
        instructions.append(self.get_length_instructions())

        if self.custom_instructions:
            instructions.append("\n## Zusätzliche Anweisungen:")
            instructions.append(self.custom_instructions)

        if self.strict_format_adherence:
            instructions.append("\n## ATTENTION: STRICT FORMAT ADHERENCE REQUIRED!")

        return "\n".join(instructions)

    def get_expected_word_range(self) -> tuple[int, int]:
        """Erwartete Wortanzahl für Qualitätsbewertung"""
        ranges = {
            TextLength.MINI_CHAT: (10, 50),
            TextLength.CHAT_CONVERSATION: (50, 150),
            TextLength.TABLE_CONVERSATION: (100, 250),
            TextLength.DETAILED_INDEPTH: (300, 800),
            TextLength.PHD_LEVEL: (800, 2000)
        }
        return ranges.get(self.text_length, (50, 200))

get_combined_instructions()

Kombiniere Format- und Längen-Anweisungen

Source code in toolboxv2/mods/isaa/base/Agent/types.py

def get_combined_instructions(self) -> str:
    """Kombiniere Format- und Längen-Anweisungen"""
    instructions = []
    instructions.append("## Format-Anforderungen:")
    instructions.append(self.get_format_instructions())
    instructions.append("\n## Längen-Anforderungen:")
    instructions.append(self.get_length_instructions())

    if self.custom_instructions:
        instructions.append("\n## Zusätzliche Anweisungen:")
        instructions.append(self.custom_instructions)

    if self.strict_format_adherence:
        instructions.append("\n## ATTENTION: STRICT FORMAT ADHERENCE REQUIRED!")

    return "\n".join(instructions)

get_expected_word_range()

Erwartete Wortanzahl für Qualitätsbewertung

Source code in toolboxv2/mods/isaa/base/Agent/types.py

def get_expected_word_range(self) -> tuple[int, int]:
    """Erwartete Wortanzahl für Qualitätsbewertung"""
    ranges = {
        TextLength.MINI_CHAT: (10, 50),
        TextLength.CHAT_CONVERSATION: (50, 150),
        TextLength.TABLE_CONVERSATION: (100, 250),
        TextLength.DETAILED_INDEPTH: (300, 800),
        TextLength.PHD_LEVEL: (800, 2000)
    }
    return ranges.get(self.text_length, (50, 200))

get_format_instructions()

Generiere Format-spezifische Anweisungen

Source code in toolboxv2/mods/isaa/base/Agent/types.py

def get_format_instructions(self) -> str:
    """Generiere Format-spezifische Anweisungen"""
    format_instructions = {
        ResponseFormat.FREE_TEXT: "Use natural continuous text without special formatting.",
        ResponseFormat.WITH_TABLES: "Integrate tables for structured data representation. Use Markdown tables.",
        ResponseFormat.WITH_BULLET_POINTS: "Structure information with bullet points (•, -, *) for better readability.",
        ResponseFormat.WITH_LISTS: "Use numbered and unnumbered lists to organize content.",
        ResponseFormat.TEXT_ONLY: "Plain text only without formatting, symbols, or structural elements.",
        ResponseFormat.MD_TEXT: "Full Markdown formatting with headings, code blocks, links, etc.",
        ResponseFormat.YAML_TEXT: "Structure responses in YAML format for machine-readable output.",
        ResponseFormat.JSON_TEXT: "Format responses as a JSON structure for API integration.",
        ResponseFormat.PSEUDO_CODE: "Use pseudocode structure for algorithmic or logical explanations.",
        ResponseFormat.CODE_STRUCTURE: "Structure like code with indentation, comments, and logical blocks."
    }
    return format_instructions.get(self.response_format, "Standard-Formatierung.")

get_length_instructions()

Generiere Längen-spezifische Anweisungen

Source code in toolboxv2/mods/isaa/base/Agent/types.py

def get_length_instructions(self) -> str:
    """Generiere Längen-spezifische Anweisungen"""
    length_instructions = {
        TextLength.MINI_CHAT: "Very short, concise answers (1–2 sentences, max 50 words). Chat style.",
        TextLength.CHAT_CONVERSATION: "Moderate conversation length (2–4 sentences, 50–150 words). Natural conversational style.",
        TextLength.TABLE_CONVERSATION: "Structured, tabular presentation with compact explanations (100–250 words).",
        TextLength.DETAILED_INDEPTH: "Comprehensive, detailed explanations (300–800 words) with depth and context.",
        TextLength.PHD_LEVEL: "Academic depth with extensive explanations (800+ words), references, and technical terminology."
    }
    return length_instructions.get(self.text_length, "Standard-Länge.")

LLMTask dataclass

Bases: Task

Spezialisierter Task für LLM-Aufrufe

Source code in toolboxv2/mods/isaa/base/Agent/types.py

@dataclass
class LLMTask(Task):
    """Spezialisierter Task für LLM-Aufrufe"""
    llm_config: dict[str, Any] = field(default_factory=lambda: {
        "model_preference": "fast",  # "fast" | "complex"
        "temperature": 0.7,
        "max_tokens": 1024
    })
    prompt_template: str = ""
    context_keys: list[str] = field(default_factory=list)  # Keys aus shared state
    output_schema: dict  = None  # JSON Schema für Validierung

PersonaConfig dataclass

Source code in toolboxv2/mods/isaa/base/Agent/types.py

@dataclass
class PersonaConfig:
    name: str
    style: str = "professional"
    personality_traits: list[str] = field(default_factory=lambda: ["helpful", "concise"])
    tone: str = "friendly"
    response_format: str = "direct"
    custom_instructions: str = ""

    format_config: FormatConfig  = None

    apply_method: str = "system_prompt"  # "system_prompt" | "post_process" | "both"
    integration_level: str = "light"  # "light" | "medium" | "heavy"

    def to_system_prompt_addition(self) -> str:
        """Convert persona to system prompt addition with format integration"""
        if self.apply_method in ["system_prompt", "both"]:
            additions = []
            additions.append(f"You are {self.name}.")
            additions.append(f"Your communication style is {self.style} with a {self.tone} tone.")

            if self.personality_traits:
                traits_str = ", ".join(self.personality_traits)
                additions.append(f"Your key traits are: {traits_str}.")

            if self.custom_instructions:
                additions.append(self.custom_instructions)

            # Format-spezifische Anweisungen hinzufügen
            if self.format_config:
                additions.append("\n" + self.format_config.get_combined_instructions())

            return " ".join(additions)
        return ""

    def update_format(self, response_format: ResponseFormat|str, text_length: TextLength|str, custom_instructions: str = ""):
        """Dynamische Format-Aktualisierung"""
        try:
            format_enum = ResponseFormat(response_format) if isinstance(response_format, str) else response_format
            length_enum = TextLength(text_length) if isinstance(text_length, str) else text_length

            if not self.format_config:
                self.format_config = FormatConfig()

            self.format_config.response_format = format_enum
            self.format_config.text_length = length_enum

            if custom_instructions:
                self.format_config.custom_instructions = custom_instructions


        except ValueError:
            raise ValueError(f"Invalid format '{response_format}' or length '{text_length}'")

    def should_post_process(self) -> bool:
        """Check if post-processing should be applied"""
        return self.apply_method in ["post_process", "both"]

should_post_process()

Check if post-processing should be applied

Source code in toolboxv2/mods/isaa/base/Agent/types.py

def should_post_process(self) -> bool:
    """Check if post-processing should be applied"""
    return self.apply_method in ["post_process", "both"]

to_system_prompt_addition()

Convert persona to system prompt addition with format integration

Source code in toolboxv2/mods/isaa/base/Agent/types.py

def to_system_prompt_addition(self) -> str:
    """Convert persona to system prompt addition with format integration"""
    if self.apply_method in ["system_prompt", "both"]:
        additions = []
        additions.append(f"You are {self.name}.")
        additions.append(f"Your communication style is {self.style} with a {self.tone} tone.")

        if self.personality_traits:
            traits_str = ", ".join(self.personality_traits)
            additions.append(f"Your key traits are: {traits_str}.")

        if self.custom_instructions:
            additions.append(self.custom_instructions)

        # Format-spezifische Anweisungen hinzufügen
        if self.format_config:
            additions.append("\n" + self.format_config.get_combined_instructions())

        return " ".join(additions)
    return ""

update_format(response_format, text_length, custom_instructions='')

Dynamische Format-Aktualisierung

Source code in toolboxv2/mods/isaa/base/Agent/types.py

def update_format(self, response_format: ResponseFormat|str, text_length: TextLength|str, custom_instructions: str = ""):
    """Dynamische Format-Aktualisierung"""
    try:
        format_enum = ResponseFormat(response_format) if isinstance(response_format, str) else response_format
        length_enum = TextLength(text_length) if isinstance(text_length, str) else text_length

        if not self.format_config:
            self.format_config = FormatConfig()

        self.format_config.response_format = format_enum
        self.format_config.text_length = length_enum

        if custom_instructions:
            self.format_config.custom_instructions = custom_instructions


    except ValueError:
        raise ValueError(f"Invalid format '{response_format}' or length '{text_length}'")

PlanData

Bases: BaseModel

Dataclass for plan data

Source code in toolboxv2/mods/isaa/base/Agent/types.py

class PlanData(BaseModel):
    """Dataclass for plan data"""
    plan_name: str = Field(..., discription="Name of the plan")
    description: str = Field(..., discription="Description of the plan")
    execution_strategy: str = Field(..., discription="Execution strategy for the plan")
    tasks: list[LLMTask | ToolTask | DecisionTask] = Field(..., discription="List of tasks in the plan")

ProgressEvent dataclass

Enhanced progress event with better error handling

Source code in toolboxv2/mods/isaa/base/Agent/types.py

@dataclass
class ProgressEvent:

    """Enhanced progress event with better error handling"""

    # === 1. Kern-Attribute (Für jedes Event) ===
    event_type: str
    node_name: str
    timestamp: float = field(default_factory=time.time)
    event_id: str = field(default_factory=lambda: str(uuid.uuid4()))
    session_id: Optional[str] = None

    # === 2. Status und Ergebnis-Attribute ===
    status: Optional[NodeStatus] = None
    success: Optional[bool] = None
    duration: Optional[float] = None
    error_details: dict[str, Any] = field(default_factory=dict)  # Strukturiert: message, type, traceback

    # === 3. LLM-spezifische Attribute ===
    llm_model: Optional[str] = None
    llm_prompt_tokens: Optional[int] = None
    llm_completion_tokens: Optional[int] = None
    llm_total_tokens: Optional[int] = None
    llm_cost: Optional[float] = None
    llm_input: Optional[Any] = None  # Optional für Debugging, kann groß sein
    llm_output: Optional[str] = None # Optional für Debugging, kann groß sein

    # === 4. Tool-spezifische Attribute ===
    tool_name: Optional[str] = None
    is_meta_tool: Optional[bool] = None
    tool_args: Optional[dict[str, Any]] = None
    tool_result: Optional[Any] = None
    tool_error: Optional[str] = None
    llm_temperature: Optional[float]  = None

    # === 5. Strategie- und Kontext-Attribute ===
    agent_name: Optional[str] = None
    task_id: Optional[str] = None
    plan_id: Optional[str] = None


    # Node/Routing data
    routing_decision: Optional[str] = None
    node_phase: Optional[str] = None
    node_duration: Optional[float] = None

    # === 6. Metadaten (Für alles andere) ===
    metadata: dict[str, Any] = field(default_factory=dict)


    def __post_init__(self):

        if self.timestamp is None:
            self.timestamp = time.time()

        if self.metadata is None:
            self.metadata = {}
        if not self.event_id:
            self.event_id = f"{self.node_name}_{self.event_type}_{int(self.timestamp * 1000000)}"
        if 'error' in self.metadata or 'error_type' in self.metadata:
            if self.error_details is None:
                self.error_details = {}
            self.error_details['error'] = self.metadata.get('error')
            self.error_details['error_type'] = self.metadata.get('error_type')
            self.status = NodeStatus.FAILED
        if self.status == NodeStatus.FAILED:
            self.success = False
        if self.status == NodeStatus.COMPLETED:
            self.success = True

    def _to_dict(self) -> dict[str, Any]:
        """Convert ProgressEvent to dictionary with proper handling of all field types"""
        result = {}

        # Get all fields from the dataclass
        for field in fields(self):
            value = getattr(self, field.name)

            # Handle None values
            if value is None:
                result[field.name] = None
                continue

            # Handle NodeStatus enum
            if isinstance(value, NodeStatus | Enum):
                result[field.name] = value.value
            # Handle dataclass objects
            elif is_dataclass(value):
                result[field.name] = asdict(value)
            # Handle dictionaries (recursively process nested enums/dataclasses)
            elif isinstance(value, dict):
                result[field.name] = self._process_dict(value)
            # Handle lists (recursively process nested items)
            elif isinstance(value, list):
                result[field.name] = self._process_list(value)
            # Handle primitive types
            else:
                result[field.name] = value

        return result

    def _process_dict(self, d: dict[str, Any]) -> dict[str, Any]:
        """Recursively process dictionary values"""
        result = {}
        for k, v in d.items():
            if isinstance(v, Enum):
                result[k] = v.value
            elif is_dataclass(v):
                result[k] = asdict(v)
            elif isinstance(v, dict):
                result[k] = self._process_dict(v)
            elif isinstance(v, list):
                result[k] = self._process_list(v)
            else:
                result[k] = v
        return result

    def _process_list(self, lst: list[Any]) -> list[Any]:
        """Recursively process list items"""
        result = []
        for item in lst:
            if isinstance(item, Enum):
                result.append(item.value)
            elif is_dataclass(item):
                result.append(asdict(item))
            elif isinstance(item, dict):
                result.append(self._process_dict(item))
            elif isinstance(item, list):
                result.append(self._process_list(item))
            else:
                result.append(item)
        return result

    @classmethod
    def from_dict(cls, data: dict[str, Any]) -> 'ProgressEvent':
        """Create ProgressEvent from dictionary"""
        # Create a copy to avoid modifying the original
        data_copy = dict(data)

        # Handle NodeStatus enum conversion from string back to enum
        if 'status' in data_copy and data_copy['status'] is not None:
            if isinstance(data_copy['status'], str):
                try:
                    data_copy['status'] = NodeStatus(data_copy['status'])
                except (ValueError, TypeError):
                    # If invalid status value, set to None
                    data_copy['status'] = None

        # Filter out any keys that aren't valid dataclass fields
        field_names = {field.name for field in fields(cls)}
        filtered_data = {k: v for k, v in data_copy.items() if k in field_names}

        # Ensure metadata is properly initialized
        if 'metadata' not in filtered_data or filtered_data['metadata'] is None:
            filtered_data['metadata'] = {}

        return cls(**filtered_data)

    def to_dict(self) -> dict[str, Any]:
        """Return event data with None values removed for compact display"""
        data = self._to_dict()

        def clean_dict(d):
            if isinstance(d, dict):
                return {k: clean_dict(v) for k, v in d.items()
                        if v is not None and v != {} and v != [] and v != ''}
            elif isinstance(d, list):
                cleaned_list = [clean_dict(item) for item in d if item is not None]
                return [item for item in cleaned_list if item != {} and item != []]
            return d

        return clean_dict(data)

    def get_chat_display_data(self) -> dict[str, Any]:
        """Get data optimized for chat view display"""
        filtered = self.filter_none_values()

        # Core fields always shown
        core_data = {
            'event_type': filtered.get('event_type'),
            'node_name': filtered.get('node_name'),
            'timestamp': filtered.get('timestamp'),
            'event_id': filtered.get('event_id'),
            'status': filtered.get('status')
        }

        # Add specific fields based on event type
        if self.event_type == 'outline_created':
            if 'metadata' in filtered:
                core_data['outline_steps'] = len(filtered['metadata'].get('outline', []))
        elif self.event_type == 'reasoning_loop':
            if 'metadata' in filtered:
                core_data.update({
                    'loop_number': filtered['metadata'].get('loop_number'),
                    'outline_step': filtered['metadata'].get('outline_step'),
                    'context_size': filtered['metadata'].get('context_size')
                })
        elif self.event_type == 'tool_call':
            core_data.update({
                'tool_name': filtered.get('tool_name'),
                'is_meta_tool': filtered.get('is_meta_tool')
            })
        elif self.event_type == 'llm_call':
            core_data.update({
                'llm_model': filtered.get('llm_model'),
                'llm_total_tokens': filtered.get('llm_total_tokens'),
                'llm_cost': filtered.get('llm_cost')
            })

        # Remove None values from core_data
        return {k: v for k, v in core_data.items() if v is not None}

    def get_detailed_display_data(self) -> dict[str, Any]:
        """Get complete filtered data for detailed popup view"""
        return self.filter_none_values()

    def get_progress_summary(self) -> str:
        """Get a brief summary for progress sidebar"""
        if self.event_type == 'reasoning_loop' and 'metadata' in self.filter_none_values():
            metadata = self.filter_none_values()['metadata']
            loop_num = metadata.get('loop_number', '?')
            step = metadata.get('outline_step', '?')
            return f"Loop {loop_num}, Step {step}"
        elif self.event_type == 'tool_call':
            tool_name = self.tool_name or 'Unknown Tool'
            return f"{'Meta ' if self.is_meta_tool else ''}{tool_name}"
        elif self.event_type == 'llm_call':
            model = self.llm_model or 'Unknown Model'
            tokens = self.llm_total_tokens
            return f"{model} ({tokens} tokens)" if tokens else model
        else:
            return self.event_type.replace('_', ' ').title()

from_dict(data) classmethod

Create ProgressEvent from dictionary

Source code in toolboxv2/mods/isaa/base/Agent/types.py

@classmethod
def from_dict(cls, data: dict[str, Any]) -> 'ProgressEvent':
    """Create ProgressEvent from dictionary"""
    # Create a copy to avoid modifying the original
    data_copy = dict(data)

    # Handle NodeStatus enum conversion from string back to enum
    if 'status' in data_copy and data_copy['status'] is not None:
        if isinstance(data_copy['status'], str):
            try:
                data_copy['status'] = NodeStatus(data_copy['status'])
            except (ValueError, TypeError):
                # If invalid status value, set to None
                data_copy['status'] = None

    # Filter out any keys that aren't valid dataclass fields
    field_names = {field.name for field in fields(cls)}
    filtered_data = {k: v for k, v in data_copy.items() if k in field_names}

    # Ensure metadata is properly initialized
    if 'metadata' not in filtered_data or filtered_data['metadata'] is None:
        filtered_data['metadata'] = {}

    return cls(**filtered_data)

get_chat_display_data()

Get data optimized for chat view display

Source code in toolboxv2/mods/isaa/base/Agent/types.py

def get_chat_display_data(self) -> dict[str, Any]:
    """Get data optimized for chat view display"""
    filtered = self.filter_none_values()

    # Core fields always shown
    core_data = {
        'event_type': filtered.get('event_type'),
        'node_name': filtered.get('node_name'),
        'timestamp': filtered.get('timestamp'),
        'event_id': filtered.get('event_id'),
        'status': filtered.get('status')
    }

    # Add specific fields based on event type
    if self.event_type == 'outline_created':
        if 'metadata' in filtered:
            core_data['outline_steps'] = len(filtered['metadata'].get('outline', []))
    elif self.event_type == 'reasoning_loop':
        if 'metadata' in filtered:
            core_data.update({
                'loop_number': filtered['metadata'].get('loop_number'),
                'outline_step': filtered['metadata'].get('outline_step'),
                'context_size': filtered['metadata'].get('context_size')
            })
    elif self.event_type == 'tool_call':
        core_data.update({
            'tool_name': filtered.get('tool_name'),
            'is_meta_tool': filtered.get('is_meta_tool')
        })
    elif self.event_type == 'llm_call':
        core_data.update({
            'llm_model': filtered.get('llm_model'),
            'llm_total_tokens': filtered.get('llm_total_tokens'),
            'llm_cost': filtered.get('llm_cost')
        })

    # Remove None values from core_data
    return {k: v for k, v in core_data.items() if v is not None}

get_detailed_display_data()

Get complete filtered data for detailed popup view

Source code in toolboxv2/mods/isaa/base/Agent/types.py

def get_detailed_display_data(self) -> dict[str, Any]:
    """Get complete filtered data for detailed popup view"""
    return self.filter_none_values()

get_progress_summary()

Get a brief summary for progress sidebar

Source code in toolboxv2/mods/isaa/base/Agent/types.py

def get_progress_summary(self) -> str:
    """Get a brief summary for progress sidebar"""
    if self.event_type == 'reasoning_loop' and 'metadata' in self.filter_none_values():
        metadata = self.filter_none_values()['metadata']
        loop_num = metadata.get('loop_number', '?')
        step = metadata.get('outline_step', '?')
        return f"Loop {loop_num}, Step {step}"
    elif self.event_type == 'tool_call':
        tool_name = self.tool_name or 'Unknown Tool'
        return f"{'Meta ' if self.is_meta_tool else ''}{tool_name}"
    elif self.event_type == 'llm_call':
        model = self.llm_model or 'Unknown Model'
        tokens = self.llm_total_tokens
        return f"{model} ({tokens} tokens)" if tokens else model
    else:
        return self.event_type.replace('_', ' ').title()

to_dict()

Return event data with None values removed for compact display

Source code in toolboxv2/mods/isaa/base/Agent/types.py

def to_dict(self) -> dict[str, Any]:
    """Return event data with None values removed for compact display"""
    data = self._to_dict()

    def clean_dict(d):
        if isinstance(d, dict):
            return {k: clean_dict(v) for k, v in d.items()
                    if v is not None and v != {} and v != [] and v != ''}
        elif isinstance(d, list):
            cleaned_list = [clean_dict(item) for item in d if item is not None]
            return [item for item in cleaned_list if item != {} and item != []]
        return d

    return clean_dict(data)

ProgressTracker

Advanced progress tracking with cost calculation

Source code in toolboxv2/mods/isaa/base/Agent/types.py

class ProgressTracker:
    """Advanced progress tracking with cost calculation"""

    def __init__(self, progress_callback: callable  = None, agent_name="unknown"):
        self.progress_callback = progress_callback
        self.events: list[ProgressEvent] = []
        self.active_timers: dict[str, float] = {}

        # Cost tracking (simplified - would need actual provider pricing)
        self.token_costs = {
            "input": 0.00001,  # $0.01/1K tokens input
            "output": 0.00003,  # $0.03/1K tokens output
        }
        self.agent_name = agent_name

    async def emit_event(self, event: ProgressEvent):
        """Emit progress event with callback and storage"""
        self.events.append(event)
        event.agent_name = self.agent_name

        if self.progress_callback:
            try:
                if asyncio.iscoroutinefunction(self.progress_callback):
                    await self.progress_callback(event)
                else:
                    self.progress_callback(event)
            except Exception:
                import traceback
                print(traceback.format_exc())


    def start_timer(self, key: str) -> float:
        """Start timing operation"""
        start_time = time.perf_counter()
        self.active_timers[key] = start_time
        return start_time

    def end_timer(self, key: str) -> float:
        """End timing operation and return duration"""
        if key not in self.active_timers:
            return 0.0
        duration = time.perf_counter() - self.active_timers[key]
        del self.active_timers[key]
        return duration

    def calculate_llm_cost(self, model: str, input_tokens: int, output_tokens: int,completion_response:Any=None) -> float:
        """Calculate approximate LLM cost"""
        cost = (input_tokens / 1000) * self.token_costs["input"] + (output_tokens / 1000) * self.token_costs["output"]
        if hasattr(completion_response, "_hidden_params"):
            cost = completion_response._hidden_params.get("response_cost", 0)
        try:
            import litellm
            cost = litellm.completion_cost(model=model, completion_response=completion_response)
        except ImportError:
            pass
        except Exception as e:
            try:
                import litellm
                cost = litellm.completion_cost(model=model.split('/')[-1], completion_response=completion_response)
            except Exception:
                pass
        return cost or (input_tokens / 1000) * self.token_costs["input"] + (output_tokens / 1000) * self.token_costs["output"]

    def get_summary(self) -> dict[str, Any]:
        """Get comprehensive progress summary"""
        summary = {
            "total_events": len(self.events),
            "llm_calls": len([e for e in self.events if e.event_type == "llm_call"]),
            "tool_calls": len([e for e in self.events if e.event_type == "tool_call"]),
            "total_cost": sum(e.llm_cost for e in self.events if e.llm_cost),
            "total_tokens": sum(e.llm_total_tokens for e in self.events if e.llm_total_tokens),
            "total_duration": sum(e.node_duration for e in self.events if e.node_duration),
            "nodes_visited": list(set(e.node_name for e in self.events)),
            "tools_used": list(set(e.tool_name for e in self.events if e.tool_name)),
            "models_used": list(set(e.llm_model for e in self.events if e.llm_model))
        }
        return summary

calculate_llm_cost(model, input_tokens, output_tokens, completion_response=None)

Calculate approximate LLM cost

Source code in toolboxv2/mods/isaa/base/Agent/types.py

def calculate_llm_cost(self, model: str, input_tokens: int, output_tokens: int,completion_response:Any=None) -> float:
    """Calculate approximate LLM cost"""
    cost = (input_tokens / 1000) * self.token_costs["input"] + (output_tokens / 1000) * self.token_costs["output"]
    if hasattr(completion_response, "_hidden_params"):
        cost = completion_response._hidden_params.get("response_cost", 0)
    try:
        import litellm
        cost = litellm.completion_cost(model=model, completion_response=completion_response)
    except ImportError:
        pass
    except Exception as e:
        try:
            import litellm
            cost = litellm.completion_cost(model=model.split('/')[-1], completion_response=completion_response)
        except Exception:
            pass
    return cost or (input_tokens / 1000) * self.token_costs["input"] + (output_tokens / 1000) * self.token_costs["output"]

emit_event(event) async

Emit progress event with callback and storage

Source code in toolboxv2/mods/isaa/base/Agent/types.py

async def emit_event(self, event: ProgressEvent):
    """Emit progress event with callback and storage"""
    self.events.append(event)
    event.agent_name = self.agent_name

    if self.progress_callback:
        try:
            if asyncio.iscoroutinefunction(self.progress_callback):
                await self.progress_callback(event)
            else:
                self.progress_callback(event)
        except Exception:
            import traceback
            print(traceback.format_exc())

end_timer(key)

End timing operation and return duration

Source code in toolboxv2/mods/isaa/base/Agent/types.py

def end_timer(self, key: str) -> float:
    """End timing operation and return duration"""
    if key not in self.active_timers:
        return 0.0
    duration = time.perf_counter() - self.active_timers[key]
    del self.active_timers[key]
    return duration

get_summary()

Get comprehensive progress summary

Source code in toolboxv2/mods/isaa/base/Agent/types.py

def get_summary(self) -> dict[str, Any]:
    """Get comprehensive progress summary"""
    summary = {
        "total_events": len(self.events),
        "llm_calls": len([e for e in self.events if e.event_type == "llm_call"]),
        "tool_calls": len([e for e in self.events if e.event_type == "tool_call"]),
        "total_cost": sum(e.llm_cost for e in self.events if e.llm_cost),
        "total_tokens": sum(e.llm_total_tokens for e in self.events if e.llm_total_tokens),
        "total_duration": sum(e.node_duration for e in self.events if e.node_duration),
        "nodes_visited": list(set(e.node_name for e in self.events)),
        "tools_used": list(set(e.tool_name for e in self.events if e.tool_name)),
        "models_used": list(set(e.llm_model for e in self.events if e.llm_model))
    }
    return summary

start_timer(key)

Start timing operation

Source code in toolboxv2/mods/isaa/base/Agent/types.py

def start_timer(self, key: str) -> float:
    """Start timing operation"""
    start_time = time.perf_counter()
    self.active_timers[key] = start_time
    return start_time

Task dataclass

Source code in toolboxv2/mods/isaa/base/Agent/types.py

@dataclass
class Task:
    id: str
    type: str
    description: str
    status: str = "pending"  # pending, running, completed, failed, paused
    priority: int = 1
    dependencies: list[str] = field(default_factory=list)
    subtasks: list[str] = field(default_factory=list)
    result: Any = None
    error: str = None
    created_at: datetime = field(default_factory=datetime.now)
    started_at: datetime  = None
    completed_at: datetime  = None
    metadata: dict[str, Any] = field(default_factory=dict)
    retry_count: int = 0
    max_retries: int = 3
    critical: bool = False

    task_identification_attr: bool = True


    def __post_init__(self):
        """Ensure all mutable defaults are properly initialized"""
        if self.metadata is None:
            self.metadata = {}
        if self.dependencies is None:
            self.dependencies = []
        if self.subtasks is None:
            self.subtasks = []

    def __getitem__(self, key):
        return getattr(self, key)

    def __setitem__(self, key, value):
        setattr(self, key, value)

__post_init__()

Ensure all mutable defaults are properly initialized

Source code in toolboxv2/mods/isaa/base/Agent/types.py

def __post_init__(self):
    """Ensure all mutable defaults are properly initialized"""
    if self.metadata is None:
        self.metadata = {}
    if self.dependencies is None:
        self.dependencies = []
    if self.subtasks is None:
        self.subtasks = []

ToolAnalysis

Bases: BaseModel

Defines the structure for a valid tool analysis.

Source code in toolboxv2/mods/isaa/base/Agent/types.py

class ToolAnalysis(BaseModel):
    """Defines the structure for a valid tool analysis."""
    primary_function: str = Field(..., description="The main purpose of the tool.")
    use_cases: list[str] = Field(..., description="Specific use cases for the tool.")
    trigger_phrases: list[str] = Field(..., description="Phrases that should trigger the tool.")
    indirect_connections: list[str] = Field(..., description="Non-obvious connections or applications.")
    complexity_scenarios: list[str] = Field(..., description="Complex scenarios where the tool can be applied.")
    user_intent_categories: list[str] = Field(..., description="Categories of user intent the tool addresses.")
    confidence_triggers: dict[str, float] = Field(..., description="Phrases mapped to confidence scores.")
    tool_complexity: str = Field(..., description="The complexity of the tool, rated as low, medium, or high.")
    args_schema: dict[str, Any] | None = Field(..., description="The schema for the tool's arguments.")

ToolTask dataclass

Bases: Task

Spezialisierter Task für Tool-Aufrufe

Source code in toolboxv2/mods/isaa/base/Agent/types.py

@dataclass
class ToolTask(Task):
    """Spezialisierter Task für Tool-Aufrufe"""
    tool_name: str = ""
    arguments: dict[str, Any] = field(default_factory=dict)  # Kann {{ }} Referenzen enthalten
    hypothesis: str = ""  # Was erwarten wir von diesem Tool?
    validation_criteria: str = ""  # Wie validieren wir das Ergebnis?
    expectation: str = ""  # Wie sollte das Ergebnis aussehen?

create_task(task_type, **kwargs)

Factory für Task-Erstellung mit korrektem Typ

Source code in toolboxv2/mods/isaa/base/Agent/types.py

def create_task(task_type: str, **kwargs) -> Task:
    """Factory für Task-Erstellung mit korrektem Typ"""
    task_classes = {
        "llm_call": LLMTask,
        "tool_call": ToolTask,
        "decision": DecisionTask,
        "generic": Task,
        "LLMTask": LLMTask,
        "ToolTask": ToolTask,
        "DecisionTask": DecisionTask,
        "Task": Task,
    }

    task_class = task_classes.get(task_type, Task)

    # Standard-Felder setzen
    if "id" not in kwargs:
        kwargs["id"] = str(uuid.uuid4())
    if "type" not in kwargs:
        kwargs["type"] = task_type
    if "critical" not in kwargs:
        kwargs["critical"] = task_type in ["llm_call", "decision"]

    # Ensure metadata is initialized
    if "metadata" not in kwargs:
        kwargs["metadata"] = {}

    # Create task and ensure post_init is called
    task = task_class(**kwargs)

    # Double-check metadata initialization
    if not hasattr(task, 'metadata') or task.metadata is None:
        task.metadata = {}

    return task

utils

LLMMessage dataclass

Represents a message in a conversation with the LLM.

Source code in toolboxv2/mods/isaa/base/Agent/utils.py

@dataclass
class LLMMessage:
    """Represents a message in a conversation with the LLM."""
    role: str  # "user", "assistant", "system", "tool"
    # Content can be string or list (e.g., multimodal with text/image dicts)
    # Conforms to LiteLLM/OpenAI structure
    content: str | list[dict[str, Any]]
    tool_call_id: str | None = None  # For tool responses
    name: str | None = None  # For tool calls/responses (function name)

    def to_dict(self) -> dict:
        """Convert to dictionary, handling potential dataclass nuances."""
        d = {"role": self.role, "content": self.content}
        if self.tool_call_id:
            d["tool_call_id"] = self.tool_call_id
        if self.name:
            d["name"] = self.name
        return d

to_dict()

Convert to dictionary, handling potential dataclass nuances.

Source code in toolboxv2/mods/isaa/base/Agent/utils.py

def to_dict(self) -> dict:
    """Convert to dictionary, handling potential dataclass nuances."""
    d = {"role": self.role, "content": self.content}
    if self.tool_call_id:
        d["tool_call_id"] = self.tool_call_id
    if self.name:
        d["name"] = self.name
    return d

WorldModel dataclass

Thread-safe representation of the agent's persistent understanding of the world.

Source code in toolboxv2/mods/isaa/base/Agent/utils.py

@dataclass
class WorldModel:
    """Thread-safe representation of the agent's persistent understanding of the world."""
    data: dict[str, Any] = dataclass_field(default_factory=dict)
    _lock: threading.Lock = dataclass_field(default_factory=threading.Lock)

    def get(self, key: str, default: Any = None) -> Any:
        with self._lock:
            return self.data.get(key, default)

    def set(self, key: str, value: Any):
        with self._lock:
            logger_wm.debug(f"WorldModel SET: {key} = {value}")
            self.data[key] = value

    def remove(self, key: str):
        with self._lock:
            if key in self.data:
                logger_wm.debug(f"WorldModel REMOVE: {key}")
                del self.data[key]

    def show(self) -> str:
        with self._lock:
            if not self.data:
                return "[empty]"
            try:
                items = [f"- {k}: {json.dumps(v, indent=None, ensure_ascii=False, default=str)}"
                         for k, v in self.data.items()]
                return "\n".join(items)
            except Exception:
                items = [f"- {k}: {str(v)}" for k, v in self.data.items()]
                return "\n".join(items)

    def to_dict(self) -> dict[str, Any]:
        with self._lock:
            # Deep copy might be needed if values are mutable and modified externally
            # For simplicity, shallow copy is used here.
            return self.data.copy()

    def update_from_dict(self, data_dict: dict[str, Any]):
        with self._lock:
            self.data.update(data_dict)
            logger_wm.debug(f"WorldModel updated from dict: {list(data_dict.keys())}")

AgentUtils

AISemanticMemory

Source code in toolboxv2/mods/isaa/base/AgentUtils.py

class AISemanticMemory(metaclass=Singleton):
    def __init__(self,
                 base_path: str = "/semantic_memory",
                 default_model: str = os.getenv("BLITZMODEL"),
                 default_embedding_model: str = os.getenv("DEFAULTMODELEMBEDDING"),
                 default_similarity_threshold: float = 0.61,
                 default_batch_size: int = 64,
                 default_n_clusters: int = 2,
                 default_deduplication_threshold: float = 0.85):
        """
        Initialize AISemanticMemory with KnowledgeBase integration

        Args:
            base_path: Root directory for memory storage
            default_model: Default model for text generation
            default_embedding_model: Default embedding model
            default_similarity_threshold: Default similarity threshold for retrieval
            default_batch_size: Default batch size for processing
            default_n_clusters: Default number of clusters for FAISS
            default_deduplication_threshold: Default threshold for deduplication
        """
        self.base_path = os.path.join(os.getcwd(), ".data", base_path)
        self.memories: dict[str, KnowledgeBase] = {}

        # Map of embedding models to their dimensions
        self.embedding_dims = {
            "text-embedding-3-small": 1536,
            "text-embedding-3-large": 3072,
            "nomic-embed-text": 768,
            "default": 768
        }

        self.default_config = {
            "embedding_model": default_embedding_model,
            "embedding_dim": self._get_embedding_dim(default_embedding_model),
            "similarity_threshold": default_similarity_threshold,
            "batch_size": default_batch_size,
            "n_clusters": default_n_clusters,
            "deduplication_threshold": default_deduplication_threshold,
            "model_name": default_model
        }

    def _get_embedding_dim(self, model_name: str) -> int:
        """Get embedding dimension for a model"""
        return self.embedding_dims.get(model_name, 768)

    @staticmethod
    def _sanitize_name(name: str) -> str:
        """Sanitize memory name for filesystem safety"""
        name = re.sub(r'[^a-zA-Z0-9_-]', '-', name)[:63].strip('-')
        if not name:
            raise ValueError("Invalid memory name")
        if len(name) < 3:
            name += "Z" * (3 - len(name))
        return name

    def create_memory(self,
                      name: str,
                      model_config: dict | None = None,
                      storage_config: dict | None = None) -> KnowledgeBase:
        """
        Create new memory store with KnowledgeBase

        Args:
            name: Unique name for the memory store
            model_config: Configuration for embedding model
            storage_config: Configuration for KnowledgeBase parameters
        """
        sanitized = self._sanitize_name(name)
        if sanitized in self.memories:
            raise ValueError(f"Memory '{name}' already exists")

        # Determine embedding model and dimension
        embedding_model = self.default_config["embedding_model"]
        model_name = self.default_config["model_name"]
        if model_config:
            embedding_model = model_config.get("embedding_model", embedding_model)
            model_name = model_config.get("model_name", model_name)
        embedding_dim = self._get_embedding_dim(embedding_model)

        # Get KnowledgeBase parameters
        kb_params = {
            "embedding_dim": embedding_dim,
            "embedding_model": embedding_model,
            "similarity_threshold": self.default_config["similarity_threshold"],
            "batch_size": self.default_config["batch_size"],
            "n_clusters": self.default_config["n_clusters"],
            "deduplication_threshold": self.default_config["deduplication_threshold"],
            "model_name": model_name,
        }

        if storage_config:
            kb_params.update({
                "similarity_threshold": storage_config.get("similarity_threshold", kb_params["similarity_threshold"]),
                "batch_size": storage_config.get("batch_size", kb_params["batch_size"]),
                "n_clusters": storage_config.get("n_clusters", kb_params["n_clusters"]),
                "model_name": storage_config.get("model_name", kb_params["model_name"]),
                "embedding_model": storage_config.get("embedding_model", kb_params["embedding_model"]),
                "deduplication_threshold": storage_config.get("deduplication_threshold",
                                                              kb_params["deduplication_threshold"]),
            })

        # Create KnowledgeBase instance
        self.memories[sanitized] = KnowledgeBase(**kb_params)
        return self.memories[sanitized]

    async def add_data(self,
                       memory_name: str,
                       data: str | list[str] | bytes | dict,
                       metadata: dict | None = None, direct=False) -> bool:
        """
        Add data to memory store

        Args:
            memory_name: Target memory store
            data: Text, list of texts, binary file, or structured data
            metadata: Optional metadata
        """
        name = self._sanitize_name(memory_name)
        kb = self.memories.get(name)
        if not kb:
            kb = self.create_memory(name)

        # Process input data
        texts = []
        if isinstance(data, bytes):
            try:
                text = extract_text_natively(data, filename="" if metadata is None else metadata.get("filename", ""))
                texts = [text.replace('\\t', '').replace('\t', '')]
            except Exception as e:
                raise ValueError(f"File processing failed: {str(e)}")
        elif isinstance(data, str):
            texts = [data.replace('\\t', '').replace('\t', '')]
        elif isinstance(data, list):
            texts = [d.replace('\\t', '').replace('\t', '') for d in data]
        elif isinstance(data, dict):
            # Custom KG not supported in current KnowledgeBase
            raise NotImplementedError("Custom knowledge graph insertion not supported")
        else:
            raise ValueError("Unsupported data type")

        # Add data to KnowledgeBase
        try:
            added, duplicates = await kb.add_data(texts, metadata, direct=direct)
            return added > 0
        except Exception as e:
            import traceback
            print(traceback.format_exc())
            raise RuntimeError(f"Data addition failed: {str(e)}")

    def get(self, names):
        return [m for n,m in self._get_target_memories(names)]

    async def query(self,
                    query: str,
                    memory_names: str | list[str] | None = None,
                    query_params: dict | None = None,
                    to_str: bool = False,
                    unified_retrieve: bool =False) -> str | list[dict]:
        """
        Query memories using KnowledgeBase retrieval

        Args:
            query: Search query
            memory_names: Target memory names
            query_params: Query parameters
            to_str: Return string format
            unified_retrieve: Unified retrieve
        """
        targets = self._get_target_memories(memory_names)
        if not targets:
            return []

        results = []
        for name, kb in targets:
            #try:
                # Use KnowledgeBase's retrieve_with_overview for comprehensive results
                result = await kb.retrieve_with_overview(
                    query=query,
                    k=query_params.get("k", 3) if query_params else 3,
                    min_similarity=query_params.get("min_similarity", 0.2) if query_params else 0.2,
                    cross_ref_depth=query_params.get("cross_ref_depth", 2) if query_params else 2,
                    max_cross_refs=query_params.get("max_cross_refs", 2) if query_params else 2,
                    max_sentences=query_params.get("max_sentences", 5) if query_params else 5
                ) if not unified_retrieve else await kb.unified_retrieve(
                    query=query,
                    k=query_params.get("k", 2) if query_params else 2,
                    min_similarity=query_params.get("min_similarity", 0.2) if query_params else 0.2,
                    cross_ref_depth=query_params.get("cross_ref_depth", 2) if query_params else 2,
                    max_cross_refs=query_params.get("max_cross_refs", 6) if query_params else 6,
                    max_sentences=query_params.get("max_sentences", 12) if query_params else 12
                )
                if result.overview:
                    results.append({
                        "memory": name,
                        "result": result
                    })
            #except Exception as e:
            #    print(f"Query failed on {name}: {str(e)}")
        if to_str:
            str_res = ""
            if not unified_retrieve:
                str_res = [
                    f"{x['memory']} - {json.dumps(x['result'].overview)}\n - {[c.text for c in x['result'].details]}\n - {[(k, [c.text for c in v]) for k, v in x['result'].cross_references.items()]}"
                    for x in results]
                # str_res =
            else:
                str_res = json.dumps(results)
            return str_res
        return results

    def _get_target_memories(self, memory_names: str | list[str] | None) -> list[tuple[str, KnowledgeBase]]:
        """Get target memories for query"""
        if not memory_names:
            return list(self.memories.items())

        names = [memory_names] if isinstance(memory_names, str) else memory_names

        targets = []
        for name in names:
            sanitized = self._sanitize_name(name)
            if kb := self.memories.get(sanitized):
                targets.append((sanitized, kb))
        return targets

    def list_memories(self) -> list[str]:
        """List all available memories"""
        return list(self.memories.keys())

    async def delete_memory(self, name: str) -> bool:
        """Delete a memory store"""
        sanitized = self._sanitize_name(name)
        if sanitized in self.memories:
            del self.memories[sanitized]
            return True
        return False

    def save_memory(self, name: str, path: str) -> bool | bytes:
        """Save a memory store to disk"""
        sanitized = self._sanitize_name(name)
        if kb := self.memories.get(sanitized):
            try:
                return kb.save(path)
            except Exception as e:
                print(f"Error saving memory: {str(e)}")
                return False
        return False

    def save_all_memories(self, path: str) -> bool:
        """Save all memory stores to disk"""
        for name, kb in self.memories.items():
            try:
                kb.save(os.path.join(path, f"{name}.pkl"))
            except Exception as e:
                print(f"Error saving memory: {str(e)}")
                return False
        return True

    def load_all_memories(self, path: str) -> bool:
        """Load all memory stores from disk"""
        for file in os.listdir(path):
            if file.endswith(".pkl"):
                try:
                    self.memories[file[:-4]] = KnowledgeBase.load(os.path.join(path, file))
                except EOFError:
                    return False
                except FileNotFoundError:
                    return False
                except Exception as e:
                    print(f"Error loading memory: {str(e)}")
                    return False
        return True

    def load_memory(self, name: str, path: str | bytes) -> bool:
        """Load a memory store from disk"""
        sanitized = self._sanitize_name(name)
        if sanitized in self.memories:
            return False
        try:
            self.memories[sanitized] = KnowledgeBase.load(path)
            return True
        except Exception:
            # print(f"Error loading memory: {str(e)}")
            return False

__init__(base_path='/semantic_memory', default_model=os.getenv('BLITZMODEL'), default_embedding_model=os.getenv('DEFAULTMODELEMBEDDING'), default_similarity_threshold=0.61, default_batch_size=64, default_n_clusters=2, default_deduplication_threshold=0.85)

Initialize AISemanticMemory with KnowledgeBase integration

Parameters:

Name	Type	Description	Default
base_path	str	Root directory for memory storage	'/semantic_memory'
default_model	str	Default model for text generation	getenv('BLITZMODEL')
default_embedding_model	str	Default embedding model	getenv('DEFAULTMODELEMBEDDING')
default_similarity_threshold	float	Default similarity threshold for retrieval	0.61
default_batch_size	int	Default batch size for processing	64
default_n_clusters	int	Default number of clusters for FAISS	2
default_deduplication_threshold	float	Default threshold for deduplication	0.85

Source code in toolboxv2/mods/isaa/base/AgentUtils.py

def __init__(self,
             base_path: str = "/semantic_memory",
             default_model: str = os.getenv("BLITZMODEL"),
             default_embedding_model: str = os.getenv("DEFAULTMODELEMBEDDING"),
             default_similarity_threshold: float = 0.61,
             default_batch_size: int = 64,
             default_n_clusters: int = 2,
             default_deduplication_threshold: float = 0.85):
    """
    Initialize AISemanticMemory with KnowledgeBase integration

    Args:
        base_path: Root directory for memory storage
        default_model: Default model for text generation
        default_embedding_model: Default embedding model
        default_similarity_threshold: Default similarity threshold for retrieval
        default_batch_size: Default batch size for processing
        default_n_clusters: Default number of clusters for FAISS
        default_deduplication_threshold: Default threshold for deduplication
    """
    self.base_path = os.path.join(os.getcwd(), ".data", base_path)
    self.memories: dict[str, KnowledgeBase] = {}

    # Map of embedding models to their dimensions
    self.embedding_dims = {
        "text-embedding-3-small": 1536,
        "text-embedding-3-large": 3072,
        "nomic-embed-text": 768,
        "default": 768
    }

    self.default_config = {
        "embedding_model": default_embedding_model,
        "embedding_dim": self._get_embedding_dim(default_embedding_model),
        "similarity_threshold": default_similarity_threshold,
        "batch_size": default_batch_size,
        "n_clusters": default_n_clusters,
        "deduplication_threshold": default_deduplication_threshold,
        "model_name": default_model
    }

add_data(memory_name, data, metadata=None, direct=False) async

Add data to memory store

Parameters:

Name	Type	Description	Default
memory_name	str	Target memory store	required
data	str | list[str] | bytes | dict	Text, list of texts, binary file, or structured data	required
metadata	dict | None	Optional metadata	None

Source code in toolboxv2/mods/isaa/base/AgentUtils.py

async def add_data(self,
                   memory_name: str,
                   data: str | list[str] | bytes | dict,
                   metadata: dict | None = None, direct=False) -> bool:
    """
    Add data to memory store

    Args:
        memory_name: Target memory store
        data: Text, list of texts, binary file, or structured data
        metadata: Optional metadata
    """
    name = self._sanitize_name(memory_name)
    kb = self.memories.get(name)
    if not kb:
        kb = self.create_memory(name)

    # Process input data
    texts = []
    if isinstance(data, bytes):
        try:
            text = extract_text_natively(data, filename="" if metadata is None else metadata.get("filename", ""))
            texts = [text.replace('\\t', '').replace('\t', '')]
        except Exception as e:
            raise ValueError(f"File processing failed: {str(e)}")
    elif isinstance(data, str):
        texts = [data.replace('\\t', '').replace('\t', '')]
    elif isinstance(data, list):
        texts = [d.replace('\\t', '').replace('\t', '') for d in data]
    elif isinstance(data, dict):
        # Custom KG not supported in current KnowledgeBase
        raise NotImplementedError("Custom knowledge graph insertion not supported")
    else:
        raise ValueError("Unsupported data type")

    # Add data to KnowledgeBase
    try:
        added, duplicates = await kb.add_data(texts, metadata, direct=direct)
        return added > 0
    except Exception as e:
        import traceback
        print(traceback.format_exc())
        raise RuntimeError(f"Data addition failed: {str(e)}")

create_memory(name, model_config=None, storage_config=None)

Create new memory store with KnowledgeBase

Parameters:

Name	Type	Description	Default
name	str	Unique name for the memory store	required
model_config	dict | None	Configuration for embedding model	None
storage_config	dict | None	Configuration for KnowledgeBase parameters	None

Source code in toolboxv2/mods/isaa/base/AgentUtils.py

def create_memory(self,
                  name: str,
                  model_config: dict | None = None,
                  storage_config: dict | None = None) -> KnowledgeBase:
    """
    Create new memory store with KnowledgeBase

    Args:
        name: Unique name for the memory store
        model_config: Configuration for embedding model
        storage_config: Configuration for KnowledgeBase parameters
    """
    sanitized = self._sanitize_name(name)
    if sanitized in self.memories:
        raise ValueError(f"Memory '{name}' already exists")

    # Determine embedding model and dimension
    embedding_model = self.default_config["embedding_model"]
    model_name = self.default_config["model_name"]
    if model_config:
        embedding_model = model_config.get("embedding_model", embedding_model)
        model_name = model_config.get("model_name", model_name)
    embedding_dim = self._get_embedding_dim(embedding_model)

    # Get KnowledgeBase parameters
    kb_params = {
        "embedding_dim": embedding_dim,
        "embedding_model": embedding_model,
        "similarity_threshold": self.default_config["similarity_threshold"],
        "batch_size": self.default_config["batch_size"],
        "n_clusters": self.default_config["n_clusters"],
        "deduplication_threshold": self.default_config["deduplication_threshold"],
        "model_name": model_name,
    }

    if storage_config:
        kb_params.update({
            "similarity_threshold": storage_config.get("similarity_threshold", kb_params["similarity_threshold"]),
            "batch_size": storage_config.get("batch_size", kb_params["batch_size"]),
            "n_clusters": storage_config.get("n_clusters", kb_params["n_clusters"]),
            "model_name": storage_config.get("model_name", kb_params["model_name"]),
            "embedding_model": storage_config.get("embedding_model", kb_params["embedding_model"]),
            "deduplication_threshold": storage_config.get("deduplication_threshold",
                                                          kb_params["deduplication_threshold"]),
        })

    # Create KnowledgeBase instance
    self.memories[sanitized] = KnowledgeBase(**kb_params)
    return self.memories[sanitized]

delete_memory(name) async

Delete a memory store

Source code in toolboxv2/mods/isaa/base/AgentUtils.py

async def delete_memory(self, name: str) -> bool:
    """Delete a memory store"""
    sanitized = self._sanitize_name(name)
    if sanitized in self.memories:
        del self.memories[sanitized]
        return True
    return False

list_memories()

List all available memories

Source code in toolboxv2/mods/isaa/base/AgentUtils.py

def list_memories(self) -> list[str]:
    """List all available memories"""
    return list(self.memories.keys())

load_all_memories(path)

Load all memory stores from disk

Source code in toolboxv2/mods/isaa/base/AgentUtils.py

def load_all_memories(self, path: str) -> bool:
    """Load all memory stores from disk"""
    for file in os.listdir(path):
        if file.endswith(".pkl"):
            try:
                self.memories[file[:-4]] = KnowledgeBase.load(os.path.join(path, file))
            except EOFError:
                return False
            except FileNotFoundError:
                return False
            except Exception as e:
                print(f"Error loading memory: {str(e)}")
                return False
    return True

load_memory(name, path)

Load a memory store from disk

Source code in toolboxv2/mods/isaa/base/AgentUtils.py

def load_memory(self, name: str, path: str | bytes) -> bool:
    """Load a memory store from disk"""
    sanitized = self._sanitize_name(name)
    if sanitized in self.memories:
        return False
    try:
        self.memories[sanitized] = KnowledgeBase.load(path)
        return True
    except Exception:
        # print(f"Error loading memory: {str(e)}")
        return False

query(query, memory_names=None, query_params=None, to_str=False, unified_retrieve=False) async

Query memories using KnowledgeBase retrieval

Parameters:

Name	Type	Description	Default
query	str	Search query	required
memory_names	str | list[str] | None	Target memory names	None
query_params	dict | None	Query parameters	None
to_str	bool	Return string format	False
unified_retrieve	bool	Unified retrieve	False

Source code in toolboxv2/mods/isaa/base/AgentUtils.py

async def query(self,
                query: str,
                memory_names: str | list[str] | None = None,
                query_params: dict | None = None,
                to_str: bool = False,
                unified_retrieve: bool =False) -> str | list[dict]:
    """
    Query memories using KnowledgeBase retrieval

    Args:
        query: Search query
        memory_names: Target memory names
        query_params: Query parameters
        to_str: Return string format
        unified_retrieve: Unified retrieve
    """
    targets = self._get_target_memories(memory_names)
    if not targets:
        return []

    results = []
    for name, kb in targets:
        #try:
            # Use KnowledgeBase's retrieve_with_overview for comprehensive results
            result = await kb.retrieve_with_overview(
                query=query,
                k=query_params.get("k", 3) if query_params else 3,
                min_similarity=query_params.get("min_similarity", 0.2) if query_params else 0.2,
                cross_ref_depth=query_params.get("cross_ref_depth", 2) if query_params else 2,
                max_cross_refs=query_params.get("max_cross_refs", 2) if query_params else 2,
                max_sentences=query_params.get("max_sentences", 5) if query_params else 5
            ) if not unified_retrieve else await kb.unified_retrieve(
                query=query,
                k=query_params.get("k", 2) if query_params else 2,
                min_similarity=query_params.get("min_similarity", 0.2) if query_params else 0.2,
                cross_ref_depth=query_params.get("cross_ref_depth", 2) if query_params else 2,
                max_cross_refs=query_params.get("max_cross_refs", 6) if query_params else 6,
                max_sentences=query_params.get("max_sentences", 12) if query_params else 12
            )
            if result.overview:
                results.append({
                    "memory": name,
                    "result": result
                })
        #except Exception as e:
        #    print(f"Query failed on {name}: {str(e)}")
    if to_str:
        str_res = ""
        if not unified_retrieve:
            str_res = [
                f"{x['memory']} - {json.dumps(x['result'].overview)}\n - {[c.text for c in x['result'].details]}\n - {[(k, [c.text for c in v]) for k, v in x['result'].cross_references.items()]}"
                for x in results]
            # str_res =
        else:
            str_res = json.dumps(results)
        return str_res
    return results

save_all_memories(path)

Save all memory stores to disk

Source code in toolboxv2/mods/isaa/base/AgentUtils.py

def save_all_memories(self, path: str) -> bool:
    """Save all memory stores to disk"""
    for name, kb in self.memories.items():
        try:
            kb.save(os.path.join(path, f"{name}.pkl"))
        except Exception as e:
            print(f"Error saving memory: {str(e)}")
            return False
    return True

save_memory(name, path)

Save a memory store to disk

Source code in toolboxv2/mods/isaa/base/AgentUtils.py

def save_memory(self, name: str, path: str) -> bool | bytes:
    """Save a memory store to disk"""
    sanitized = self._sanitize_name(name)
    if kb := self.memories.get(sanitized):
        try:
            return kb.save(path)
        except Exception as e:
            print(f"Error saving memory: {str(e)}")
            return False
    return False

PyEnvEval

Source code in toolboxv2/mods/isaa/base/AgentUtils.py

class PyEnvEval:
    def __init__(self):
        self.local_env = locals().copy()
        self.global_env = {'local_env': self.local_env}  # globals().copy()

    def eval_code(self, code):
        try:
            exec(code, self.global_env, self.local_env)
            result = eval(code, self.global_env, self.local_env)
            return self.format_output(result)
        except Exception as e:
            return self.format_output(str(e))

    def get_env(self):
        local_env_str = self.format_env(self.local_env)
        return f'Locals:\n{local_env_str}'

    @staticmethod
    def format_output(output):
        return f'Ergebnis: {output}'

    @staticmethod
    def format_env(env):
        return '\n'.join(f'{key}: {value}' for key, value in env.items())

    def run_and_display(self, python_code):
        """function to eval python code"""
        start = f'Start-state:\n{self.get_env()}'
        result = self.eval_code(python_code)
        end = f'End-state:\n{self.get_env()}'
        return f'{start}\nResult:\n{result}\n{end}'

    def tool(self):
        return {"PythonEval": {"func": self.run_and_display, "description": "Use Python Code to Get to an Persis Answer! input must be valid python code all non code parts must be comments!"}}

run_and_display(python_code)

function to eval python code

Source code in toolboxv2/mods/isaa/base/AgentUtils.py

def run_and_display(self, python_code):
    """function to eval python code"""
    start = f'Start-state:\n{self.get_env()}'
    result = self.eval_code(python_code)
    end = f'End-state:\n{self.get_env()}'
    return f'{start}\nResult:\n{result}\n{end}'

anything_from_str_to_dict(data, expected_keys=None, mini_task=lambda x: '')

Versucht, einen String in ein oder mehrere Dictionaries umzuwandeln. Berücksichtigt dabei die erwarteten Schlüssel und ihre Standardwerte.

Source code in toolboxv2/mods/isaa/base/AgentUtils.py

def anything_from_str_to_dict(data: str, expected_keys: dict = None, mini_task=lambda x: ''):
    """
    Versucht, einen String in ein oder mehrere Dictionaries umzuwandeln.
    Berücksichtigt dabei die erwarteten Schlüssel und ihre Standardwerte.
    """
    if len(data) < 4:
        return []

    if expected_keys is None:
        expected_keys = {}

    result = []
    json_objects = find_json_objects_in_str(data)
    if not json_objects and data.startswith('[') and data.endswith(']'):
        json_objects = eval(data)
    if json_objects and len(json_objects) > 0 and isinstance(json_objects[0], dict):
        result.extend([{**expected_keys, **ob} for ob in json_objects])
    if not result:
        completed_object = complete_json_object(data, mini_task)
        if completed_object is not None:
            result.append(completed_object)
    if len(result) == 0 and expected_keys:
        result = [{list(expected_keys.keys())[0]: data}]
    for res in result:
        if isinstance(res, list) and len(res) > 0:
            res = res[0]
        for key, value in expected_keys.items():
            if key not in res:
                res[key] = value

    if len(result) == 0:
        fixed = fix_json(data)
        if fixed:
            result.append(fixed)

    return result

complete_json_object(data, mini_task)

Ruft eine Funktion auf, um einen String in das richtige Format zu bringen. Gibt das resultierende JSON-Objekt zurück, wenn die Funktion erfolgreich ist, sonst None.

Source code in toolboxv2/mods/isaa/base/AgentUtils.py

def complete_json_object(data: str, mini_task):
    """
    Ruft eine Funktion auf, um einen String in das richtige Format zu bringen.
    Gibt das resultierende JSON-Objekt zurück, wenn die Funktion erfolgreich ist, sonst None.
    """
    ret = mini_task(
        f"Vervollständige das Json Object. Und bringe den string in das Richtige format. data={data}\nJson=")
    if ret:
        return anything_from_str_to_dict(ret)
    return None

detect_shell()

Detects the best available shell and the argument to execute a command. Returns: A tuple of (shell_executable, command_argument). e.g., ('/bin/bash', '-c') or ('powershell.exe', '-Command')

Source code in toolboxv2/mods/isaa/base/AgentUtils.py

def detect_shell() -> tuple[str, str]:
    """
    Detects the best available shell and the argument to execute a command.
    Returns:
        A tuple of (shell_executable, command_argument).
        e.g., ('/bin/bash', '-c') or ('powershell.exe', '-Command')
    """
    if platform.system() == "Windows":
        if shell_path := shutil.which("pwsh"):
            return shell_path, "-Command"
        if shell_path := shutil.which("powershell"):
            return shell_path, "-Command"
        return "cmd.exe", "/c"

    shell_env = os.environ.get("SHELL")
    if shell_env and shutil.which(shell_env):
        return shell_env, "-c"

    for shell in ["bash", "zsh", "sh"]:
        if shell_path := shutil.which(shell):
            return shell_path, "-c"

    return "/bin/sh", "-c"

extract_text_natively(data, filename='')

Extrahiert Text aus verschiedenen Dateitypen mit nativen Python-Methoden oder reinen Python-Bibliotheken (speziell PyPDF2 für PDFs).

Parameters:

Name	Type	Description	Default
data	bytes	Der Inhalt der Datei als Bytes.	required
filename	str	Der Originaldateiname, um den Typ zu bestimmen.	''

Returns:

Name	Type	Description
str	str	Der extrahierte Text.

Raises:

Type	Description
ValueError	Wenn der Dateityp nicht unterstützt wird oder die Verarbeitung fehlschlägt.
ImportError	Wenn PyPDF2 für die Verarbeitung von PDF-Dateien benötigt, aber nicht installiert ist.

Source code in toolboxv2/mods/isaa/base/AgentUtils.py

def extract_text_natively(data: bytes, filename: str = "") -> str:
    """
    Extrahiert Text aus verschiedenen Dateitypen mit nativen Python-Methoden
    oder reinen Python-Bibliotheken (speziell PyPDF2 für PDFs).

    Args:
        data (bytes): Der Inhalt der Datei als Bytes.
        filename (str, optional): Der Originaldateiname, um den Typ zu bestimmen.

    Returns:
        str: Der extrahierte Text.

    Raises:
        ValueError: Wenn der Dateityp nicht unterstützt wird oder die Verarbeitung fehlschlägt.
        ImportError: Wenn PyPDF2 für die Verarbeitung von PDF-Dateien benötigt, aber nicht installiert ist.
    """
    file_ext = filename.lower().split('.')[-1] if '.' in filename else ''

    # 1. DOCX-Verarbeitung (nativ mit zipfile und xml)
    if data.startswith(b'PK\x03\x04'):
        try:
            docx_file = io.BytesIO(data)
            text_parts = []
            with zipfile.ZipFile(docx_file) as zf:
                namespace = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}"
                body_path = "word/document.xml"
                if body_path in zf.namelist():
                    xml_content = zf.read(body_path)
                    tree = ET.fromstring(xml_content)
                    for para in tree.iter(f"{namespace}p"):
                        texts_in_para = [node.text for node in para.iter(f"{namespace}t") if node.text]
                        if texts_in_para:
                            text_parts.append("".join(texts_in_para))
                return "\n".join(text_parts)
        except (zipfile.BadZipFile, ET.ParseError):
            pass  # Fährt fort, falls es eine ZIP-Datei, aber kein gültiges DOCX ist

    # 2. PDF-Verarbeitung (mit PyPDF2)
    if data.startswith(b'%PDF-'):
        if PyPDF2 is None:
            raise ImportError(
                "Die Bibliothek 'PyPDF2' wird benötigt, um PDF-Dateien zu verarbeiten. Bitte installieren Sie sie mit 'pip install PyPDF2'.")

        try:
            # Erstelle ein In-Memory-Dateiobjekt für PyPDF2
            pdf_file = io.BytesIO(data)
            # Verwende PdfFileReader aus PyPDF2
            pdf_reader = PyPDF2.PdfFileReader(pdf_file)

            text_parts = []
            # Iteriere durch die Seiten
            for page_num in range(pdf_reader.numPages):
                page = pdf_reader.getPage(page_num)
                # Extrahiere Text mit extractText()
                page_text = page.extractText()
                if page_text:
                    text_parts.append(page_text)

            return "\n".join(text_parts)
        except Exception as e:
            raise ValueError(f"PDF-Verarbeitung mit PyPDF2 fehlgeschlagen: {e}")

    # 3. Fallback auf reinen Text (TXT)

    try:
        return data.decode('utf-8')
    except UnicodeDecodeError:
        try:
            return data.decode('latin-1')
        except Exception as e:
            raise ValueError(f"Text-Dekodierung fehlgeschlagen: {e}")

find_json_objects_in_str(data)

Sucht nach JSON-Objekten innerhalb eines Strings. Gibt eine Liste von JSON-Objekten zurück, die im String gefunden wurden.

Source code in toolboxv2/mods/isaa/base/AgentUtils.py

def find_json_objects_in_str(data: str):
    """
    Sucht nach JSON-Objekten innerhalb eines Strings.
    Gibt eine Liste von JSON-Objekten zurück, die im String gefunden wurden.
    """
    json_objects = extract_json_objects(data)
    if not isinstance(json_objects, list):
        json_objects = [json_objects]
    return [get_json_from_json_str(ob, 10) for ob in json_objects if get_json_from_json_str(ob, 10) is not None]

get_json_from_json_str(json_str, repeat=1)

Versucht, einen JSON-String in ein Python-Objekt umzuwandeln.

Wenn beim Parsen ein Fehler auftritt, versucht die Funktion, das Problem zu beheben, indem sie das Zeichen an der Position des Fehlers durch ein Escape-Zeichen ersetzt. Dieser Vorgang wird bis zu repeat-mal wiederholt.

Parameters:

Name	Type	Description	Default
json_str	str or list or dict	Der JSON-String, der geparst werden soll.	required
repeat	int	Die Anzahl der Versuche, das Parsen durchzuführen.	1

Returns:

Type	Description
dict or None	Das resultierende Python-Objekt.

Source code in toolboxv2/mods/isaa/base/AgentUtils.py

def get_json_from_json_str(json_str: str or list or dict, repeat: int = 1) -> dict or None:
    """Versucht, einen JSON-String in ein Python-Objekt umzuwandeln.

    Wenn beim Parsen ein Fehler auftritt, versucht die Funktion, das Problem zu beheben,
    indem sie das Zeichen an der Position des Fehlers durch ein Escape-Zeichen ersetzt.
    Dieser Vorgang wird bis zu `repeat`-mal wiederholt.

    Args:
        json_str: Der JSON-String, der geparst werden soll.
        repeat: Die Anzahl der Versuche, das Parsen durchzuführen.

    Returns:
        Das resultierende Python-Objekt.
    """
    for _ in range(repeat):
        try:
            return parse_json_with_auto_detection(json_str)
        except json.JSONDecodeError as e:
            unexp = int(re.findall(r'\(char (\d+)\)', str(e))[0])
            unesc = json_str.rfind(r'"', 0, unexp)
            json_str = json_str[:unesc] + r'\"' + json_str[unesc + 1:]
            closg = json_str.find(r'"', unesc + 2)
            json_str = json_str[:closg] + r'\"' + json_str[closg + 1:]
        new = fix_json_object(json_str)
        if new is not None:
            json_str = new
    get_logger().info(f"Unable to parse JSON string after {json_str}")
    return None

parse_json_with_auto_detection(json_data)

Parses JSON data, automatically detecting if a value is a JSON string and parsing it accordingly. If a value cannot be parsed as JSON, it is returned as is.

Source code in toolboxv2/mods/isaa/base/AgentUtils.py

def parse_json_with_auto_detection(json_data):
    """
    Parses JSON data, automatically detecting if a value is a JSON string and parsing it accordingly.
    If a value cannot be parsed as JSON, it is returned as is.
    """

    def try_parse_json(value):
        """
        Tries to parse a value as JSON. If the parsing fails, the original value is returned.
        """
        try:
            # print("parse_json_with_auto_detection:", type(value), value)
            parsed_value = json.loads(value)
            # print("parsed_value:", type(parsed_value), parsed_value)
            # If the parsed value is a string, it might be a JSON string, so we try to parse it again
            if isinstance(parsed_value, str):
                return eval(parsed_value)
            else:
                return parsed_value
        except Exception:
            # logging.warning(f"Failed to parse value as JSON: {value}. Exception: {e}")
            return value

    get_logger()

    if isinstance(json_data, dict):
        return {key: parse_json_with_auto_detection(value) for key, value in json_data.items()}
    elif isinstance(json_data, list):
        return [parse_json_with_auto_detection(item) for item in json_data]
    else:
        return try_parse_json(json_data)

KnowledgeBase

Chunk dataclass

Represents a chunk of text with its embedding and metadata

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py

@dataclass(slots=True)
class Chunk:
    """Represents a chunk of text with its embedding and metadata"""
    text: str
    embedding: np.ndarray
    metadata: dict[str, Any]
    content_hash: str
    cluster_id: int | None = None

ConceptAnalysis

Bases: BaseModel

Represents the analysis of key concepts.

Attributes:

Name	Type	Description
key_concepts	list[str]	A list of primary key concepts identified.
relationships	list[str]	A list of relationships between the identified key concepts.
importance_hierarchy	list[str]	A list that represents the hierarchical importance of the key concepts.

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py

class ConceptAnalysis(BaseModel):
    """
    Represents the analysis of key concepts.

    Attributes:
        key_concepts (list[str]): A list of primary key concepts identified.
        relationships (list[str]): A list of relationships between the identified key concepts.
        importance_hierarchy (list[str]): A list that represents the hierarchical importance of the key concepts.
    """
    key_concepts: list[str]
    relationships: list[str]
    importance_hierarchy: list[str]

ConceptExtractor

Handles extraction of concepts and relationships from text

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py

class ConceptExtractor:
    """Handles extraction of concepts and relationships from text"""

    def __init__(self, knowledge_base, requests_per_second = 85.):
        self.kb = knowledge_base
        self.concept_graph = ConceptGraph()
        self.requests_per_second = requests_per_second

    async def extract_concepts(self, texts: list[str], metadatas: list[dict[str, Any]]) -> list[list[Concept]]:
        """
        Extract concepts from texts using concurrent processing with rate limiting.
        Requests are made at the specified rate while responses are processed asynchronously.
        """
        # Ensure metadatas list matches texts length
        metadatas = metadatas + [{}] * (len(texts) - len(metadatas))

        # Initialize rate limiter
        rate_limiter = DynamicRateLimiter()

        system_prompt = (
            "Analyze the given text and extract key concepts and their relationships. For each concept:\n"
            "1. Identify the concept name and category (technical, domain, method, property, ...)\n"
            "2. Determine relationships with other concepts (uses, part_of, similar_to, depends_on, ...)\n"
            "3. Assess importance (0-1 score) based on centrality to the text\n"
            "4. Extract relevant context snippets\n"
            "5. Max 5 Concepts!\n"
            "only return in json format!\n"
            """{"concepts": [{
                "name": "concept_name",
                "category": "category_name",
                "relationships": {
                    "relationship_type": ["related_concept1", "related_concept2"]
                },
                "importance_score": 0.0,
                "context_snippets": ["relevant text snippet"]
            }]}\n"""
        )

        # Prepare all requests
        requests = [
            (idx, f"Text to Convert in to JSON structure:\n{text}", system_prompt, metadata)
            for idx, (text, metadata) in enumerate(zip(texts, metadatas, strict=False))
        ]

        async def process_single_request(idx: int, prompt: str, system_prompt: str, metadata: dict[str, Any]):
            """Process a single request with rate limiting"""
            try:
                from toolboxv2.mods.isaa.extras.adapter import litellm_complete
                # Wait for rate limit
                await rate_limiter.acquire()
                i__[1] += 1
                # Make API call without awaiting the response
                response_future = litellm_complete(
                    prompt=prompt,
                    system_prompt=system_prompt,
                    response_format=Concepts,
                    model_name=self.kb.model_name,
                    fallbacks=["groq/gemma2-9b-it"] +
                              [m for m in os.getenv("FALLBACKS_MODELS_PREM", '').split(',') if m]
                )

                return idx, response_future

            except Exception as e:
                print(f"Error initiating request {idx}: {str(e)}")
                return idx, None

        async def process_response(idx: int, response_future) -> list[Concept]:
            """Process the response once it's ready"""
            try:
                if response_future is None:
                    return []

                response = await response_future
                return await self._process_response(response, metadatas[idx])

            except Exception as e:
                print(f"Error processing response {idx}: {str(e)}")
                return []

        # Create tasks for all requests
        request_tasks = []
        batch_size = self.kb.batch_size

        rate_limiter.update_rate(self.requests_per_second)

        for batch_start in range(0, len(requests), batch_size):
            batch = requests[batch_start:batch_start + batch_size]

            # Create tasks for the batch
            batch_tasks = [
                process_single_request(idx, prompt, sys_prompt, meta)
                for idx, prompt, sys_prompt, meta in batch
            ]
            request_tasks.extend(batch_tasks)

        # Execute all requests with rate limiting
        request_results = await asyncio.gather(*request_tasks)

        # Process responses as they complete
        response_tasks = [
            process_response(idx, response_future)
            for idx, response_future in request_results
        ]

        # Gather all results
        all_results = await asyncio.gather(*response_tasks)

        # Sort results by original index
        sorted_results = [[] for _ in texts]
        for idx, concepts in enumerate(all_results):
            sorted_results[idx] = concepts

        return sorted_results

    async def _process_response(self, response: Any, metadata: dict[str, Any]) -> list[Concept]:
        """Helper method to process a single response and convert it to Concepts"""
        try:
            # Extract content from response
            if hasattr(response, 'choices'):
                content = response.choices[0].message.content
                if content is None:
                    content = response.choices[0].message.tool_calls[0].function.arguments
                if content is None:
                    return []
            elif isinstance(response, str):
                content = response
            else:
                print(f"Unexpected response type: {type(response)}")
                return []

            from toolboxv2.mods.isaa.extras.filter import after_format
            # Parse JSON and create concepts
            concept_data = after_format(content)
            concepts = []

            for concept_info in concept_data.get("concepts", []):
                concept = Concept(
                    name=concept_info["name"],
                    category=concept_info.get("category", "N/A"),
                    relationships={k: set(v) for k, v in concept_info.get("relationships", {}).items()},
                    importance_score=concept_info.get("importance_score", 0.1),
                    context_snippets=concept_info.get("context_snippets", "N/A"),
                    metadata=metadata
                )
                concepts.append(concept)
                self.concept_graph.add_concept(concept)

            return concepts

        except Exception:
            i__[2] +=1
            return []

    async def process_chunks(self, chunks: list[Chunk]) -> None:
        """
        Process all chunks in batch to extract and store concepts.
        Each chunk's metadata will be updated with the concept names and relationships.
        """
        # Gather all texts from the chunks.
        texts = [chunk.text for chunk in chunks]
        # Call extract_concepts once with all texts.
        all_concepts = await self.extract_concepts(texts, [chunk.metadata for chunk in chunks])

        # Update each chunk's metadata with its corresponding concepts.
        for chunk, concepts in zip(chunks, all_concepts, strict=False):
            chunk.metadata["concepts"] = [c.name for c in concepts]
            chunk.metadata["concept_relationships"] = {
                c.name: {k: list(v) for k, v in c.relationships.items()}
                for c in concepts
            }

    async def query_concepts(self, query: str) -> dict[str, any]:
        """Query the concept graph based on natural language query"""

        system_prompt = """
        Convert the natural language query about concepts into a structured format that specifies:
        1. Main concepts of interest
        2. Desired relationship types
        3. Any category filters
        4. Importance threshold

        Format as JSON.
        """

        prompt = f"""
        Query: {query}

        Convert to this JSON structure:
        {{
            "target_concepts": ["concept1", "concept2"],
            "relationship_types": ["type1", "type2"],
            "categories": ["category1", "category2"],
            "min_importance": 0.0
        }}
        """

        try:
            from toolboxv2.mods.isaa.extras.adapter import litellm_complete
            response = await litellm_complete(
                model_name=self.kb.model_name,
                prompt=prompt,
                system_prompt=system_prompt,
                response_format=TConcept
            )

            query_params = json.loads(response)

            results = {
                "concepts": {},
                "relationships": [],
                "groups": []
            }

            # Find matching concepts
            for concept_name in query_params["target_concepts"]:
                if concept_name in self.concept_graph.concepts:
                    concept = self.concept_graph.concepts[concept_name]
                    if concept.importance_score >= query_params["min_importance"]:
                        results["concepts"][concept_name] = {
                            "category": concept.category,
                            "importance": concept.importance_score,
                            "context": concept.context_snippets
                        }

                        # Get relationships
                        for rel_type in query_params["relationship_types"]:
                            related = self.concept_graph.get_related_concepts(
                                concept_name, rel_type
                            )
                            for related_concept in related:
                                results["relationships"].append({
                                    "from": concept_name,
                                    "to": related_concept,
                                    "type": rel_type
                                })

            # Group concepts by category
            category_groups = defaultdict(list)
            for concept_name, concept_info in results["concepts"].items():
                category_groups[concept_info["category"]].append(concept_name)
            results["groups"] = [
                {"category": cat, "concepts": concepts}
                for cat, concepts in category_groups.items()
            ]

            return results

        except Exception as e:
            print(f"Error querying concepts: {str(e)}")
            return {"concepts": {}, "relationships": [], "groups": []}

extract_concepts(texts, metadatas) async

Extract concepts from texts using concurrent processing with rate limiting. Requests are made at the specified rate while responses are processed asynchronously.

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py

async def extract_concepts(self, texts: list[str], metadatas: list[dict[str, Any]]) -> list[list[Concept]]:
    """
    Extract concepts from texts using concurrent processing with rate limiting.
    Requests are made at the specified rate while responses are processed asynchronously.
    """
    # Ensure metadatas list matches texts length
    metadatas = metadatas + [{}] * (len(texts) - len(metadatas))

    # Initialize rate limiter
    rate_limiter = DynamicRateLimiter()

    system_prompt = (
        "Analyze the given text and extract key concepts and their relationships. For each concept:\n"
        "1. Identify the concept name and category (technical, domain, method, property, ...)\n"
        "2. Determine relationships with other concepts (uses, part_of, similar_to, depends_on, ...)\n"
        "3. Assess importance (0-1 score) based on centrality to the text\n"
        "4. Extract relevant context snippets\n"
        "5. Max 5 Concepts!\n"
        "only return in json format!\n"
        """{"concepts": [{
            "name": "concept_name",
            "category": "category_name",
            "relationships": {
                "relationship_type": ["related_concept1", "related_concept2"]
            },
            "importance_score": 0.0,
            "context_snippets": ["relevant text snippet"]
        }]}\n"""
    )

    # Prepare all requests
    requests = [
        (idx, f"Text to Convert in to JSON structure:\n{text}", system_prompt, metadata)
        for idx, (text, metadata) in enumerate(zip(texts, metadatas, strict=False))
    ]

    async def process_single_request(idx: int, prompt: str, system_prompt: str, metadata: dict[str, Any]):
        """Process a single request with rate limiting"""
        try:
            from toolboxv2.mods.isaa.extras.adapter import litellm_complete
            # Wait for rate limit
            await rate_limiter.acquire()
            i__[1] += 1
            # Make API call without awaiting the response
            response_future = litellm_complete(
                prompt=prompt,
                system_prompt=system_prompt,
                response_format=Concepts,
                model_name=self.kb.model_name,
                fallbacks=["groq/gemma2-9b-it"] +
                          [m for m in os.getenv("FALLBACKS_MODELS_PREM", '').split(',') if m]
            )

            return idx, response_future

        except Exception as e:
            print(f"Error initiating request {idx}: {str(e)}")
            return idx, None

    async def process_response(idx: int, response_future) -> list[Concept]:
        """Process the response once it's ready"""
        try:
            if response_future is None:
                return []

            response = await response_future
            return await self._process_response(response, metadatas[idx])

        except Exception as e:
            print(f"Error processing response {idx}: {str(e)}")
            return []

    # Create tasks for all requests
    request_tasks = []
    batch_size = self.kb.batch_size

    rate_limiter.update_rate(self.requests_per_second)

    for batch_start in range(0, len(requests), batch_size):
        batch = requests[batch_start:batch_start + batch_size]

        # Create tasks for the batch
        batch_tasks = [
            process_single_request(idx, prompt, sys_prompt, meta)
            for idx, prompt, sys_prompt, meta in batch
        ]
        request_tasks.extend(batch_tasks)

    # Execute all requests with rate limiting
    request_results = await asyncio.gather(*request_tasks)

    # Process responses as they complete
    response_tasks = [
        process_response(idx, response_future)
        for idx, response_future in request_results
    ]

    # Gather all results
    all_results = await asyncio.gather(*response_tasks)

    # Sort results by original index
    sorted_results = [[] for _ in texts]
    for idx, concepts in enumerate(all_results):
        sorted_results[idx] = concepts

    return sorted_results

process_chunks(chunks) async

Process all chunks in batch to extract and store concepts. Each chunk's metadata will be updated with the concept names and relationships.

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py

async def process_chunks(self, chunks: list[Chunk]) -> None:
    """
    Process all chunks in batch to extract and store concepts.
    Each chunk's metadata will be updated with the concept names and relationships.
    """
    # Gather all texts from the chunks.
    texts = [chunk.text for chunk in chunks]
    # Call extract_concepts once with all texts.
    all_concepts = await self.extract_concepts(texts, [chunk.metadata for chunk in chunks])

    # Update each chunk's metadata with its corresponding concepts.
    for chunk, concepts in zip(chunks, all_concepts, strict=False):
        chunk.metadata["concepts"] = [c.name for c in concepts]
        chunk.metadata["concept_relationships"] = {
            c.name: {k: list(v) for k, v in c.relationships.items()}
            for c in concepts
        }

query_concepts(query) async

Query the concept graph based on natural language query

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py

async def query_concepts(self, query: str) -> dict[str, any]:
    """Query the concept graph based on natural language query"""

    system_prompt = """
    Convert the natural language query about concepts into a structured format that specifies:
    1. Main concepts of interest
    2. Desired relationship types
    3. Any category filters
    4. Importance threshold

    Format as JSON.
    """

    prompt = f"""
    Query: {query}

    Convert to this JSON structure:
    {{
        "target_concepts": ["concept1", "concept2"],
        "relationship_types": ["type1", "type2"],
        "categories": ["category1", "category2"],
        "min_importance": 0.0
    }}
    """

    try:
        from toolboxv2.mods.isaa.extras.adapter import litellm_complete
        response = await litellm_complete(
            model_name=self.kb.model_name,
            prompt=prompt,
            system_prompt=system_prompt,
            response_format=TConcept
        )

        query_params = json.loads(response)

        results = {
            "concepts": {},
            "relationships": [],
            "groups": []
        }

        # Find matching concepts
        for concept_name in query_params["target_concepts"]:
            if concept_name in self.concept_graph.concepts:
                concept = self.concept_graph.concepts[concept_name]
                if concept.importance_score >= query_params["min_importance"]:
                    results["concepts"][concept_name] = {
                        "category": concept.category,
                        "importance": concept.importance_score,
                        "context": concept.context_snippets
                    }

                    # Get relationships
                    for rel_type in query_params["relationship_types"]:
                        related = self.concept_graph.get_related_concepts(
                            concept_name, rel_type
                        )
                        for related_concept in related:
                            results["relationships"].append({
                                "from": concept_name,
                                "to": related_concept,
                                "type": rel_type
                            })

        # Group concepts by category
        category_groups = defaultdict(list)
        for concept_name, concept_info in results["concepts"].items():
            category_groups[concept_info["category"]].append(concept_name)
        results["groups"] = [
            {"category": cat, "concepts": concepts}
            for cat, concepts in category_groups.items()
        ]

        return results

    except Exception as e:
        print(f"Error querying concepts: {str(e)}")
        return {"concepts": {}, "relationships": [], "groups": []}

ConceptGraph

Manages concept relationships and hierarchies

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py

class ConceptGraph:
    """Manages concept relationships and hierarchies"""

    def __init__(self):
        self.concepts: dict[str, Concept] = {}

    def add_concept(self, concept: Concept):
        """Add or update a concept in the graph"""
        if concept.name.lower() in self.concepts:
            # Merge relationships and context
            existing = self.concepts[concept.name.lower()]
            for rel_type, related in concept.relationships.items():
                if rel_type not in existing.relationships:
                    existing.relationships[rel_type] = set()
                existing.relationships[rel_type].update(related)
            existing.context_snippets.extend(concept.context_snippets)
            # Update importance score with rolling average
            existing.importance_score = (existing.importance_score + concept.importance_score) / 2
        else:
            self.concepts[concept.name.lower()] = concept

    def get_related_concepts(self, concept_name: str, relationship_type: str | None = None) -> set[str]:
        """Get related concepts, optionally filtered by relationship type"""
        if concept_name not in self.concepts:
            return set()

        concept = self.concepts[concept_name.lower()]
        if relationship_type:
            return concept.relationships.get(relationship_type, set())

        related = set()
        for relations in concept.relationships.values():
            related.update(relations)
        return related


    def convert_to_networkx(self) -> nx.DiGraph:
        """Convert ConceptGraph to NetworkX graph with layout"""
        print(f"Converting to NetworkX graph with {len(self.concepts.values())} concepts")

        G = nx.DiGraph()

        if len(self.concepts.values()) == 0:
            return G

        for concept in self.concepts.values():
            cks = '\n - '.join(concept.context_snippets[:4])
            G.add_node(
                concept.name,
                size=concept.importance_score * 10,
                group=concept.category,
                title=f"""
                    {concept.name}
                    Category: {concept.category}
                    Importance: {concept.importance_score:.2f}
                    Context: \n - {cks}
                    """
            )

            for rel_type, targets in concept.relationships.items():
                for target in targets:
                    G.add_edge(concept.name, target, label=rel_type, title=rel_type)

        return G

add_concept(concept)

Add or update a concept in the graph

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py

def add_concept(self, concept: Concept):
    """Add or update a concept in the graph"""
    if concept.name.lower() in self.concepts:
        # Merge relationships and context
        existing = self.concepts[concept.name.lower()]
        for rel_type, related in concept.relationships.items():
            if rel_type not in existing.relationships:
                existing.relationships[rel_type] = set()
            existing.relationships[rel_type].update(related)
        existing.context_snippets.extend(concept.context_snippets)
        # Update importance score with rolling average
        existing.importance_score = (existing.importance_score + concept.importance_score) / 2
    else:
        self.concepts[concept.name.lower()] = concept

convert_to_networkx()

Convert ConceptGraph to NetworkX graph with layout

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py

def convert_to_networkx(self) -> nx.DiGraph:
    """Convert ConceptGraph to NetworkX graph with layout"""
    print(f"Converting to NetworkX graph with {len(self.concepts.values())} concepts")

    G = nx.DiGraph()

    if len(self.concepts.values()) == 0:
        return G

    for concept in self.concepts.values():
        cks = '\n - '.join(concept.context_snippets[:4])
        G.add_node(
            concept.name,
            size=concept.importance_score * 10,
            group=concept.category,
            title=f"""
                {concept.name}
                Category: {concept.category}
                Importance: {concept.importance_score:.2f}
                Context: \n - {cks}
                """
        )

        for rel_type, targets in concept.relationships.items():
            for target in targets:
                G.add_edge(concept.name, target, label=rel_type, title=rel_type)

    return G

get_related_concepts(concept_name, relationship_type=None)

Get related concepts, optionally filtered by relationship type

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py

def get_related_concepts(self, concept_name: str, relationship_type: str | None = None) -> set[str]:
    """Get related concepts, optionally filtered by relationship type"""
    if concept_name not in self.concepts:
        return set()

    concept = self.concepts[concept_name.lower()]
    if relationship_type:
        return concept.relationships.get(relationship_type, set())

    related = set()
    for relations in concept.relationships.values():
        related.update(relations)
    return related

Concepts

Bases: BaseModel

Represents a collection of key concepts.

Attributes:

Name	Type	Description
concepts	List[rConcept]	A list of Concept instances, each representing an individual key concept.

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py

class Concepts(BaseModel):
    """
    Represents a collection of key concepts.

    Attributes:
        concepts (List[rConcept]): A list of Concept instances, each representing an individual key concept.
    """
    concepts: list[rConcept]

DataModel

Bases: BaseModel

The main data model that encapsulates the overall analysis.

Attributes:

Name	Type	Description
main_summary	str	A Detailed overview summarizing the key findings and relations format MD string.
concept_analysis	ConceptAnalysis	An instance containing the analysis of key concepts.
topic_insights	TopicInsights	An instance containing insights regarding the topics.
relevance_assessment	RelevanceAssessment	An instance assessing the relevance and alignment of the query.

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py

class DataModel(BaseModel):
    """
    The main data model that encapsulates the overall analysis.

    Attributes:
        main_summary (str): A Detailed overview summarizing the key findings and relations format MD string.
        concept_analysis (ConceptAnalysis): An instance containing the analysis of key concepts.
        topic_insights (TopicInsights): An instance containing insights regarding the topics.
        relevance_assessment (RelevanceAssessment): An instance assessing the relevance and alignment of the query.
    """
    main_summary: str
    concept_analysis: ConceptAnalysis
    topic_insights: TopicInsights
    relevance_assessment: RelevanceAssessment

DynamicRateLimiter

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py

class DynamicRateLimiter:
    def __init__(self):
        self.last_request_time = 0.0
        self._lock = asyncio.Lock()

    def update_rate(self, requests_per_second: float):
        """Update rate limit dynamically"""
        self.min_interval = 1.0 / requests_per_second if requests_per_second > 0 else float('inf')

    async def acquire(self):
        """Acquire permission to make a request"""
        async with self._lock:
            current_time = time.time()
            time_since_last = current_time - self.last_request_time
            if time_since_last < self.min_interval:
                wait_time = self.min_interval - time_since_last
                await asyncio.sleep(wait_time)
            self.last_request_time = time.time()

acquire() async

Acquire permission to make a request

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py

async def acquire(self):
    """Acquire permission to make a request"""
    async with self._lock:
        current_time = time.time()
        time_since_last = current_time - self.last_request_time
        if time_since_last < self.min_interval:
            wait_time = self.min_interval - time_since_last
            await asyncio.sleep(wait_time)
        self.last_request_time = time.time()

update_rate(requests_per_second)

Update rate limit dynamically

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py

def update_rate(self, requests_per_second: float):
    """Update rate limit dynamically"""
    self.min_interval = 1.0 / requests_per_second if requests_per_second > 0 else float('inf')

GraphVisualizer

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py

class GraphVisualizer:
    @staticmethod
    def visualize(nx_graph: nx.DiGraph, output_file: str = "concept_graph.html", get_output=False):
        """Create interactive visualization using PyVis"""
        from pyvis.network import Network
        net = Network(
            height="800px",
            width="100%",
            notebook=False,
            directed=True,
            bgcolor="#1a1a1a",
            font_color="white"
        )

        net.from_nx(nx_graph)

        net.save_graph(output_file)
        print(f"Graph saved to {output_file} Open in browser to view.", len(nx_graph))
        if get_output:
            c = open(output_file, encoding="utf-8").read()
            os.remove(output_file)
            return c

visualize(nx_graph, output_file='concept_graph.html', get_output=False) staticmethod

Create interactive visualization using PyVis

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py

@staticmethod
def visualize(nx_graph: nx.DiGraph, output_file: str = "concept_graph.html", get_output=False):
    """Create interactive visualization using PyVis"""
    from pyvis.network import Network
    net = Network(
        height="800px",
        width="100%",
        notebook=False,
        directed=True,
        bgcolor="#1a1a1a",
        font_color="white"
    )

    net.from_nx(nx_graph)

    net.save_graph(output_file)
    print(f"Graph saved to {output_file} Open in browser to view.", len(nx_graph))
    if get_output:
        c = open(output_file, encoding="utf-8").read()
        os.remove(output_file)
        return c

KnowledgeBase

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py

class KnowledgeBase:
    def __init__(self, embedding_dim: int = 256, similarity_threshold: float = 0.61, batch_size: int = 12,
                 n_clusters: int = 4, deduplication_threshold: float = 0.85, model_name=os.getenv("SUMMARYMODEL"),
                 embedding_model=os.getenv("DEFAULTMODELEMBEDDING"),
                 vis_class:str | None = "FaissVectorStore",
                 vis_kwargs:dict[str, Any] | None=None,
                 requests_per_second=85.,
                 chunk_size: int = 3600,
                 chunk_overlap: int = 130,
                 separator: str = "\n"
                 ):
        """Initialize the knowledge base with given parameters"""

        self.existing_hashes: set[str] = set()
        self.embedding_model = embedding_model
        self.embedding_dim = embedding_dim
        self.similarity_threshold = similarity_threshold
        self.deduplication_threshold = deduplication_threshold
        if model_name == "openrouter/mistralai/mistral-nemo":
            batch_size = 9
            requests_per_second = 1.5
        self.batch_size = batch_size
        self.n_clusters = n_clusters
        self.model_name = model_name
        self.sto: list = []

        self.text_splitter = TextSplitter(chunk_size=chunk_size,chunk_overlap=chunk_overlap, separator=separator)
        self.similarity_graph = {}
        self.concept_extractor = ConceptExtractor(self, requests_per_second)

        self.vis_class = None
        self.vis_kwargs = None
        self.vdb = None
        self.init_vis(vis_class, vis_kwargs)

    def init_vis(self, vis_class, vis_kwargs):
        if vis_class is None:
            vis_class = "FaissVectorStore"
        if vis_class == "FaissVectorStore":
            if vis_kwargs is None:
                vis_kwargs = {
                    "dimension": self.embedding_dim
                }
            self.vdb = FaissVectorStore(**vis_kwargs)
        else:
            from toolboxv2.mods.isaa.base.VectorStores.taichiNumpyNumbaVectorStores import (
                EnhancedVectorStore,
                FastVectorStore1,
                FastVectorStoreO,
                NumpyVectorStore,
                VectorStoreConfig,
            )
        if vis_class == "FastVectorStoreO":
            if vis_kwargs is None:
                vis_kwargs = {
                    "embedding_size": self.embedding_dim
                }
            self.vdb = FastVectorStoreO(**vis_kwargs)
        if vis_class == "EnhancedVectorStore":
            if vis_kwargs is None:
                vis_kwargs = {
                    "dimension": self.embedding_dim
                }
            vis_kwargs = VectorStoreConfig(**vis_kwargs)
            self.vdb = EnhancedVectorStore(vis_kwargs)
        if vis_class == "FastVectorStore1":
            self.vdb = FastVectorStore1()
        if vis_class == "NumpyVectorStore":
            self.vdb = NumpyVectorStore()

        self.vis_class = vis_class
        self.vis_kwargs = vis_kwargs


    @staticmethod
    def compute_hash(text: str) -> str:
        """Compute SHA-256 hash of text"""
        return hashlib.sha256(text.encode('utf-8', errors='ignore')).hexdigest()

    async def _get_embeddings(self, texts: list[str]) -> np.ndarray:
        """Get normalized embeddings in batches"""
        try:
            async def process_batch(batch: list[str]) -> np.ndarray:
                from toolboxv2.mods.isaa.extras.adapter import litellm_embed
                # print("Processing", batch)
                embeddings = await litellm_embed(texts=batch, model=self.embedding_model, dimensions=self.embedding_dim)
                return normalize_vectors(embeddings)

            tasks = []
            for i in range(0, len(texts), self.batch_size):
                batch = texts[i:i + self.batch_size]
                tasks.append(process_batch(batch))

            embeddings = await asyncio.gather(*tasks)
            i__[0] += len(texts)
            return np.vstack(embeddings)
        except Exception as e:
            get_logger().error(f"Error generating embeddings: {str(e)}")
            raise



    def _remove_similar_chunks(self, threshold: float = None) -> int:
        """Remove chunks that are too similar to each other"""
        if len(self.vdb.chunks) < 2:
            return 0

        if threshold is None:
            threshold = self.deduplication_threshold

        try:
            # Get all embeddings
            embeddings = np.vstack([c.embedding for c in self.vdb.chunks])
            n = len(embeddings)

            # Compute similarity matrix
            similarities = np.dot(embeddings, embeddings.T)

            # Create mask for chunks to keep
            keep_mask = np.ones(n, dtype=bool)

            # Iterate through chunks
            for i in range(n):
                if not keep_mask[i]:
                    continue

                # Find chunks that are too similar to current chunk
                similar_indices = similarities[i] >= threshold
                similar_indices[i] = False  # Don't count self-similarity

                # Mark similar chunks for removal
                keep_mask[similar_indices] = False

            # Keep only unique chunks
            unique_chunks = [chunk for chunk, keep in zip(self.vdb.chunks, keep_mask, strict=False) if keep]
            removed_count = len(self.vdb.chunks) - len(unique_chunks)

            # Update chunks and hashes
            self.vdb.chunks = unique_chunks
            self.existing_hashes = {chunk.content_hash for chunk in self.vdb.chunks}

            # Rebuild index if chunks were removed
            if removed_count > 0:
                self.vdb.rebuild_index()


            return removed_count

        except Exception as e:
            get_logger().error(f"Error removing similar chunks: {str(e)}")
            raise

    async def _add_data(
        self,
        texts: list[str],
        metadata: list[dict[str, Any]] | None= None,
    ) -> tuple[int, int]:
        """
        Process and add new data to the knowledge base
        Returns: Tuple of (added_count, duplicate_count)
        """
        if len(texts) == 0:
            return -1, -1
        try:
            # Compute hashes and filter exact duplicates
            hashes = [self.compute_hash(text) for text in texts]
            unique_data = []
            for t, m, h in zip(texts, metadata, hashes, strict=False):
                if h in self.existing_hashes:
                    continue
                # Update existing hashes
                self.existing_hashes.add(h)
                unique_data.append((t, m, h))

            if not unique_data:
                return 0, len(texts)

            # Get embeddings
            embeddings = await self._get_embeddings(texts)

            texts = []
            metadata = []
            hashes = []
            embeddings_final = []
            if len(self.vdb.chunks):
                for i, d in enumerate(unique_data):
                    c = self.vdb.search(embeddings[i], 5, self.deduplication_threshold)
                    if len(c) > 2:
                        continue
                    t, m, h = d
                    texts.append(t)
                    metadata.append(m)
                    hashes.append(h)
                    embeddings_final.append(embeddings[i])

            else:
                texts , metadata, hashes = zip(*unique_data, strict=False)
                embeddings_final = embeddings

            if not texts:  # All were similar to existing chunks
                return 0, len(unique_data)

            # Create and add new chunks
            new_chunks = [
                Chunk(text=t, embedding=e, metadata=m, content_hash=h)
                for t, e, m, h in zip(texts, embeddings_final, metadata, hashes, strict=False)
            ]

            # Add new chunks
            # Update index
            if new_chunks:
                all_embeddings = np.vstack([c.embedding for c in new_chunks])
                self.vdb.add_embeddings(all_embeddings, new_chunks)

            # Remove similar chunks from the entire collection
            removed = self._remove_similar_chunks()
            get_logger().info(f"Removed {removed} similar chunks during deduplication")
            # Invalidate visualization cache

            if len(new_chunks) - removed > 0:
                # Process new chunks for concepts
                await self.concept_extractor.process_chunks(new_chunks)
            print("[total, calls, errors]", i__)

            return len(new_chunks) - removed, len(texts) - len(new_chunks) + removed

        except Exception as e:
            get_logger().error(f"Error adding data: {str(e)}")
            raise


    async def add_data(
        self,
        texts: list[str],
        metadata: list[dict[str, Any]] | None = None, direct:bool = False
    ) -> tuple[int, int]:
        """Enhanced version with smart splitting and clustering"""
        if isinstance(texts, str):
            texts = [texts]
        if metadata is None:
            metadata = [{}] * len(texts)
        if isinstance(metadata, dict):
            metadata = [metadata]
        if len(texts) != len(metadata):
            raise ValueError("Length of texts and metadata must match")

        if not direct and len(texts) == 1 and len(texts[0]) < 10_000:
            if len(self.sto) < self.batch_size and len(texts) == 1:
                self.sto.append((texts[0], metadata[0]))
                return -1, -1
            if len(self.sto) >= self.batch_size:
                _ = [texts.append(t) or metadata.append([m]) for (t, m) in self.sto]
                self.sto = []

        # Split large texts
        split_texts = []
        split_metadata = []

        while Spinner("Saving Data to Memory", symbols='t'):

            for idx, text in enumerate(texts):
                chunks = self.text_splitter.split_text(text)
                split_texts.extend(chunks)

                # Adjust metadata for splits
                meta = metadata[idx] if metadata else {}
                if isinstance(meta, list):
                    meta = meta[0]
                for i, _chunk in enumerate(chunks):
                    chunk_meta = meta.copy()
                    chunk_meta.update({
                        'chunk_index': i,
                        'total_chunks': len(chunks),
                        'original_text_id': idx
                    })
                    split_metadata.append(chunk_meta)

            return await self._add_data(split_texts, split_metadata)

    def _update_similarity_graph(self, embeddings: np.ndarray, chunk_ids: list[int]):
        """Update similarity graph for connected information detection"""
        similarities = np.dot(embeddings, embeddings.T)

        for i in range(len(chunk_ids)):
            for j in range(i + 1, len(chunk_ids)):
                if similarities[i, j] >= self.similarity_threshold:
                    id1, id2 = chunk_ids[i], chunk_ids[j]
                    if id1 not in self.similarity_graph:
                        self.similarity_graph[id1] = set()
                    if id2 not in self.similarity_graph:
                        self.similarity_graph[id2] = set()
                    self.similarity_graph[id1].add(id2)
                    self.similarity_graph[id2].add(id1)

    async def retrieve(
        self,
        query: str="",
        query_embedding: np.ndarray | None = None,
        k: int = 5,
        min_similarity: float = 0.2,
        include_connected: bool = True
    ) -> list[Chunk]:
        """Enhanced retrieval with connected information"""
        if query_embedding is None:
            query_embedding = (await self._get_embeddings([query]))[0]
        k = min(k, len(self.vdb.chunks))
        if k <= 0:
            return []
        initial_results = self.vdb.search(query_embedding, k, min_similarity)

        if not include_connected or not initial_results:
            return initial_results

        # Find connected chunks
        connected_chunks = set()
        for chunk in initial_results:
            chunk_id = self.vdb.chunks.index(chunk)
            if chunk_id in self.similarity_graph:
                connected_chunks.update(self.similarity_graph[chunk_id])

        # Add connected chunks to results
        all_chunks = self.vdb.chunks
        additional_results = [all_chunks[i] for i in connected_chunks
                              if all_chunks[i] not in initial_results]

        # Sort by similarity to query
        all_results = initial_results + additional_results

        return sorted(
            all_results,
            key=lambda x: np.dot(x.embedding, query_embedding),
            reverse=True
        )[:k * 2]  # Return more results when including connected information

    async def forget_irrelevant(self, irrelevant_concepts: list[str], similarity_threshold: float | None=None) -> int:
        """
        Remove chunks similar to irrelevant concepts
        Returns: Number of chunks removed
        """
        if not irrelevant_concepts:
            return 0

        if similarity_threshold is None:
            similarity_threshold = self.similarity_threshold

        try:
            irrelevant_embeddings = await self._get_embeddings(irrelevant_concepts)
            initial_count = len(self.vdb.chunks)

            def is_relevant(chunk: Chunk) -> bool:
                similarities = np.dot(chunk.embedding, irrelevant_embeddings.T)
                do_keep = np.max(similarities) < similarity_threshold
                if do_keep:
                    return True
                for c in chunk.metadata.get("concepts", []):
                    if c in self.concept_extractor.concept_graph.concepts:
                        del self.concept_extractor.concept_graph.concepts[c]
                return False

            relevant_chunks = [chunk for chunk in self.vdb.chunks if is_relevant(chunk)]
            self.vdb.chunks = relevant_chunks
            self.existing_hashes = {chunk.content_hash for chunk in self.vdb.chunks}
            self.vdb.rebuild_index()

            return initial_count - len(self.vdb.chunks)

        except Exception as e:
            get_logger().error(f"Error forgetting irrelevant concepts: {str(e)}")
            raise

    ## ----------------------------------------------------------------

    def _cluster_chunks(
        self,
        chunks: list[Chunk],
        query_embedding: np.ndarray | None = None,
        min_cluster_size: int = 2,
        min_samples: int = 1,
        max_clusters: int = 10
    ) -> dict[int, list[Chunk]]:
        """
        Enhanced clustering of chunks into topics with query awareness
        and dynamic parameter adjustment
        """
        if len(chunks) < 2:
            return {0: chunks}

        embeddings = np.vstack([chunk.embedding for chunk in chunks])

        # Normalize embeddings for cosine similarity
        embeddings = normalize_vectors(embeddings)

        # If query is provided, weight embeddings by query relevance
        if query_embedding is not None:
            query_similarities = np.dot(embeddings, query_embedding)
            # Apply soft weighting to maintain structure while considering query relevance
            embeddings = embeddings * query_similarities[:, np.newaxis]
            embeddings = normalize_vectors(embeddings)

        # Dynamic parameter adjustment based on dataset size
        adjusted_min_cluster_size = max(
            min_cluster_size,
            min(len(chunks) // 10, 5)  # Scale with data size, max 5
        )

        adjusted_min_samples = max(
            min_samples,
            adjusted_min_cluster_size // 2
        )

        # Try different parameter combinations for optimal clustering
        best_clusters = None
        best_score = float('-inf')

        epsilon_range = [0.2, 0.3, 0.4]
        try:
            HDBSCAN = __import__('sklearn.cluster').HDBSCAN
        except:
            print("install scikit-learn pip install scikit-learn for better results")
            return self._fallback_clustering(chunks, query_embedding)

        for epsilon in epsilon_range:
            clusterer = HDBSCAN(
                min_cluster_size=adjusted_min_cluster_size,
                min_samples=adjusted_min_samples,
                metric='cosine',
                cluster_selection_epsilon=epsilon
            )

            cluster_labels = clusterer.fit_predict(embeddings)

            # Skip if all points are noise
            if len(set(cluster_labels)) <= 1:
                continue

            # Calculate clustering quality metrics
            score = self._evaluate_clustering(
                embeddings,
                cluster_labels,
                query_embedding
            )

            if score > best_score:
                best_score = score
                best_clusters = cluster_labels

        # If no good clustering found, fall back to simpler approach
        if best_clusters is None:
            return self._fallback_clustering(chunks, query_embedding)

        # Organize chunks by cluster
        clusters: dict[int, list[Chunk]] = {}

        # Sort clusters by size and relevance
        cluster_scores = []

        for label in set(best_clusters):
            if label == -1:  # Handle noise points separately
                continue

            # Fixed: Use boolean mask to select chunks for current cluster
            cluster_mask = best_clusters == label
            cluster_chunks = [chunk for chunk, is_in_cluster in zip(chunks, cluster_mask, strict=False) if is_in_cluster]

            # Skip empty clusters
            if not cluster_chunks:
                continue

            # Calculate cluster score based on size and query relevance
            score = len(cluster_chunks)
            if query_embedding is not None:
                cluster_embeddings = np.vstack([c.embedding for c in cluster_chunks])
                query_relevance = np.mean(np.dot(cluster_embeddings, query_embedding))
                score = score * (1 + query_relevance)  # Boost by relevance

            cluster_scores.append((label, score, cluster_chunks))

        # Sort clusters by score and limit to max_clusters
        cluster_scores.sort(key=lambda x: x[1], reverse=True)

        # Assign cleaned clusters
        for i, (_, _, cluster_chunks) in enumerate(cluster_scores[:max_clusters]):
            clusters[i] = cluster_chunks

        # Handle noise points by assigning to nearest cluster
        noise_chunks = [chunk for chunk, label in zip(chunks, best_clusters, strict=False) if label == -1]
        if noise_chunks:
            self._assign_noise_points(noise_chunks, clusters, query_embedding)

        return clusters

    @staticmethod
    def _evaluate_clustering(
        embeddings: np.ndarray,
        labels: np.ndarray,
        query_embedding: np.ndarray | None = None
    ) -> float:
        """
        Evaluate clustering quality using multiple metrics
        """
        if len(set(labels)) <= 1:
            return float('-inf')

        # Calculate silhouette score for cluster cohesion
        try:
            sil_score = __import__('sklearn.metrics').silhouette_score(embeddings, labels, metric='cosine')
        except:
            print("install scikit-learn pip install scikit-learn for better results")
            sil_score = 0

        # Calculate Davies-Bouldin score for cluster separation
        try:
            db_score = -__import__('sklearn.metrics').davies_bouldin_score(embeddings, labels)  # Negated as lower is better
        except:
            print("install scikit-learn pip install scikit-learn for better results")
            db_score = 0

        # Calculate query relevance if provided
        query_score = 0
        if query_embedding is not None:
            unique_labels = set(labels) - {-1}
            if unique_labels:
                query_sims = []
                for label in unique_labels:
                    cluster_mask = labels == label
                    cluster_embeddings = embeddings[cluster_mask]
                    cluster_centroid = np.mean(cluster_embeddings, axis=0)
                    query_sims.append(np.dot(cluster_centroid, query_embedding))
                query_score = np.mean(query_sims)

        # Combine scores with weights
        combined_score = (
            0.4 * sil_score +
            0.3 * db_score +
            0.3 * query_score
        )

        return combined_score

    @staticmethod
    def _fallback_clustering(
        chunks: list[Chunk],
        query_embedding: np.ndarray | None = None
    ) -> dict[int, list[Chunk]]:
        """
        Simple fallback clustering when HDBSCAN fails
        """
        if query_embedding is not None:
            # Sort by query relevance
            chunks_with_scores = [
                (chunk, np.dot(chunk.embedding, query_embedding))
                for chunk in chunks
            ]
            chunks_with_scores.sort(key=lambda x: x[1], reverse=True)
            chunks = [c for c, _ in chunks_with_scores]

        # Create fixed-size clusters
        clusters = {}
        cluster_size = max(2, len(chunks) // 5)

        for i in range(0, len(chunks), cluster_size):
            clusters[len(clusters)] = chunks[i:i + cluster_size]

        return clusters

    @staticmethod
    def _assign_noise_points(
        noise_chunks: list[Chunk],
        clusters: dict[int, list[Chunk]],
        query_embedding: np.ndarray | None = None
    ) -> None:
        """
        Assign noise points to nearest clusters
        """
        if not clusters:
            clusters[0] = noise_chunks
            return

        for chunk in noise_chunks:
            best_cluster = None
            best_similarity = float('-inf')

            for cluster_id, cluster_chunks in clusters.items():
                cluster_embeddings = np.vstack([c.embedding for c in cluster_chunks])
                cluster_centroid = np.mean(cluster_embeddings, axis=0)

                similarity = np.dot(chunk.embedding, cluster_centroid)

                # Consider query relevance in assignment if available
                if query_embedding is not None:
                    query_sim = np.dot(chunk.embedding, query_embedding)
                    similarity = 0.7 * similarity + 0.3 * query_sim

                if similarity > best_similarity:
                    best_similarity = similarity
                    best_cluster = cluster_id

            if best_cluster is not None:
                clusters[best_cluster].append(chunk)

    @staticmethod
    def _generate_topic_summary(
        chunks: list[Chunk],
        query_embedding: np.ndarray,
        max_sentences=3
    ) -> str:
        """Generate a summary for a topic using most representative chunks"""
        if not chunks:
            return ""

        # Find chunks most similar to cluster centroid
        embeddings = np.vstack([chunk.embedding for chunk in chunks])
        centroid = embeddings.mean(axis=0)

        # Calculate similarities to both centroid and query
        centroid_sims = np.dot(embeddings, centroid)
        query_sims = np.dot(embeddings, query_embedding)

        # Combine both similarities
        combined_sims = 0.7 * centroid_sims + 0.3 * query_sims

        # Select top sentences from most representative chunks
        top_indices = np.argsort(combined_sims)[-max_sentences:]
        summary_chunks = [chunks[i] for i in top_indices]

        # Extract key sentences
        sentences = []
        for chunk in summary_chunks:
            sentences.extend(sent.strip() for sent in chunk.text.split('.') if sent.strip())

        return '. '.join(sentences[:max_sentences]) + '.'

    async def retrieve_with_overview(
        self,
        query: str,
        query_embedding=None,
        k: int = 5,
        min_similarity: float = 0.2,
        max_sentences: int = 5,
        cross_ref_depth: int = 2,
        max_cross_refs: int = 10  # New parameter to control cross-reference count
    ) -> RetrievalResult:
        """Enhanced retrieval with better cross-reference handling"""
        # Get initial results with query embedding
        if query_embedding is None:
            query_embedding = (await self._get_embeddings([query]))[0]
        initial_results = await self.retrieve(query_embedding=query_embedding, k=k, min_similarity=min_similarity)

        if not initial_results:
            return RetrievalResult([], [], {})

        # Find cross-references with similarity scoring
        initial_ids = {self.vdb.chunks.index(chunk) for chunk in initial_results}
        related_ids = self._find_cross_references(
            initial_ids,
            depth=cross_ref_depth,
            query_embedding=query_embedding  # Pass query embedding for relevance scoring
        )

        # Get all relevant chunks with smarter filtering
        all_chunks = self.vdb.chunks
        all_relevant_chunks = initial_results + [
            chunk for i, chunk in enumerate(all_chunks)
            if i in related_ids and self._is_relevant_cross_ref(
                chunk,
                query_embedding,
                initial_results
            )
        ]

        # Enhanced clustering with dynamic cluster size
        clusters = self._cluster_chunks(
            all_relevant_chunks,
            query_embedding=query_embedding
        )

        # Fallback: If no clusters are found, treat all relevant chunks as a single cluster.
        if not clusters:
            print("No clusters found. Falling back to using all relevant chunks as a single cluster.")
            clusters = {0: all_relevant_chunks}

        # Generate summaries and organize results
        overview = []
        cross_references = {}

        for cluster_id, cluster_chunks in clusters.items():
            summary = self._generate_topic_summary(
                cluster_chunks,
                query_embedding,
                max_sentences=max_sentences  # Increased for more context
            )

            # Enhanced chunk sorting with combined scoring
            sorted_chunks = self._sort_chunks_by_relevance(
                cluster_chunks,
                query_embedding,
                initial_results
            )

            # Separate direct matches and cross-references
            direct_matches_ = [{'text':c.text, 'metadata':c.metadata} for c in sorted_chunks if c in initial_results]
            direct_matches = []
            for match in direct_matches_:
                if match in direct_matches:
                    continue
                direct_matches.append(match)
            cross_refs_ = [c for c in sorted_chunks if c not in initial_results]
            cross_refs = []
            for match in cross_refs_:
                if match in cross_refs:
                    continue
                cross_refs.append(match)
            # Limit cross-references while maintaining diversity
            selected_cross_refs = self._select_diverse_cross_refs(
                cross_refs,
                max_cross_refs,
                query_embedding
            )

            topic_info = {
                'topic_id': cluster_id,
                'summary': summary,
                'main_chunks': [x for x in direct_matches[:3]],
                'chunk_count': len(cluster_chunks),
                'relevance_score': self._calculate_topic_relevance(
                    cluster_chunks,
                    query_embedding
                )
            }
            overview.append(topic_info)

            if selected_cross_refs:
                cross_references[f"topic_{cluster_id}"] = selected_cross_refs

        # Sort overview by relevance score
        overview.sort(key=lambda x: x['relevance_score'], reverse=True)

        return RetrievalResult(
            overview=overview,
            details=initial_results,
            cross_references=cross_references
        )

    def _find_cross_references(
        self,
        chunk_ids: set[int],
        depth: int,
        query_embedding: np.ndarray
    ) -> set[int]:
        """Enhanced cross-reference finding with relevance scoring"""
        related_ids = set(chunk_ids)
        current_depth = 0
        frontier = set(chunk_ids)

        while current_depth < depth and frontier:
            new_frontier = set()
            for chunk_id in frontier:
                if chunk_id in self.similarity_graph:
                    # Score potential cross-references by relevance
                    candidates = self.similarity_graph[chunk_id] - related_ids
                    scored_candidates = [
                        (cid, self._calculate_topic_relevance(
                            [self.vdb.chunks[cid]],
                            query_embedding
                        ))
                        for cid in candidates
                    ]

                    # Filter by relevance threshold
                    relevant_candidates = {
                        cid for cid, score in scored_candidates
                        if score > 0.5  # Adjustable threshold
                    }
                    new_frontier.update(relevant_candidates)

            related_ids.update(new_frontier)
            frontier = new_frontier
            current_depth += 1

        return related_ids

    @staticmethod
    def _is_relevant_cross_ref(
        chunk: Chunk,
        query_embedding: np.ndarray,
        initial_results: list[Chunk]
    ) -> bool:
        """Determine if a cross-reference is relevant enough to include"""
        # Calculate similarity to query
        query_similarity = np.dot(chunk.embedding, query_embedding)

        # Calculate similarity to initial results
        initial_similarities = [
            np.dot(chunk.embedding, r.embedding) for r in initial_results
        ]
        max_initial_similarity = max(initial_similarities)

        # Combined relevance score
        relevance_score = 0.7 * query_similarity + 0.3 * max_initial_similarity

        return relevance_score > 0.6  # Adjustable threshold

    @staticmethod
    def _select_diverse_cross_refs(
        cross_refs: list[Chunk],
        max_count: int,
        query_embedding: np.ndarray
    ) -> list[Chunk]:
        """Select diverse and relevant cross-references"""
        if not cross_refs or len(cross_refs) <= max_count:
            return cross_refs

        # Calculate diversity scores
        embeddings = np.vstack([c.embedding for c in cross_refs])
        similarities = np.dot(embeddings, embeddings.T)

        selected = []
        remaining = list(enumerate(cross_refs))

        while len(selected) < max_count and remaining:
            # Score remaining chunks by relevance and diversity
            scores = []
            for idx, chunk in remaining:
                relevance = np.dot(chunk.embedding, query_embedding)
                diversity = 1.0
                if selected:
                    # Calculate diversity penalty based on similarity to selected chunks
                    selected_similarities = [
                        similarities[idx][list(cross_refs).index(s)]
                        for s in selected
                    ]
                    diversity = 1.0 - max(selected_similarities)

                combined_score = 0.7 * relevance + 0.3 * diversity
                scores.append((combined_score, idx, chunk))

            # Select the highest scoring chunk
            scores.sort(reverse=True)
            _, idx, chunk = scores[0]
            selected.append(chunk)
            remaining = [(i, c) for i, c in remaining if i != idx]

        return selected

    @staticmethod
    def _calculate_topic_relevance(
        chunks: list[Chunk],
        query_embedding: np.ndarray,
    ) -> float:
        """Calculate overall topic relevance score"""
        if not chunks:
            return 0.0

        similarities = [
            np.dot(chunk.embedding, query_embedding) for chunk in chunks
        ]
        return np.mean(similarities)

    @staticmethod
    def _sort_chunks_by_relevance(
        chunks: list[Chunk],
        query_embedding: np.ndarray,
        initial_results: list[Chunk]
    ) -> list[Chunk]:
        """Sort chunks by combined relevance score"""
        scored_chunks = []
        for chunk in chunks:
            query_similarity = np.dot(chunk.embedding, query_embedding)
            initial_similarities = [
                np.dot(chunk.embedding, r.embedding)
                for r in initial_results
            ]
            max_initial_similarity = max(initial_similarities) if initial_similarities else 0

            # Combined score favoring query relevance
            combined_score = 0.7 * query_similarity + 0.3 * max_initial_similarity
            scored_chunks.append((combined_score, chunk))

        scored_chunks.sort(reverse=True)
        return [chunk for _, chunk in scored_chunks]

    async def query_concepts(self, query: str) -> dict[str, any]:
        """Query concepts extracted from the knowledge base"""
        return await self.concept_extractor.query_concepts(query)

    async def unified_retrieve(
        self,
        query: str,
        k: int = 5,
        min_similarity: float = 0.2,
        cross_ref_depth: int = 2,
        max_cross_refs: int = 10,
        max_sentences: int = 10
    ) -> dict[str, Any]:
        """
        Unified retrieval function that combines concept querying, retrieval with overview,
        and basic retrieval, then generates a comprehensive summary using LLM.

        Args:
            query: Search query string
            k: Number of primary results to retrieve
            min_similarity: Minimum similarity threshold for retrieval
            cross_ref_depth: Depth for cross-reference search
            max_cross_refs: Maximum number of cross-references per topic
            max_sentences: Maximum number Sentences in the main summary text

        Returns:
            Dictionary containing comprehensive results including summary and details
        """
        # Get concept information
        concept_results = await self.concept_extractor.query_concepts(query)

        # Get retrieval overview

        query_embedding = (await self._get_embeddings([query]))[0]
        overview_results = await self.retrieve_with_overview(
            query=query,
            query_embedding=query_embedding,
            k=k,
            min_similarity=min_similarity,
            cross_ref_depth=cross_ref_depth,
            max_cross_refs=max_cross_refs,
            max_sentences=max_sentences
        )

        # Get basic retrieval results
        basic_results = await self.retrieve(
            query_embedding=query_embedding,
            k=k,
            min_similarity=min_similarity
        )
        if len(basic_results) == 0:
            return {}
        if len(basic_results) == 1 and isinstance(basic_results[0], str) and basic_results[0].endswith('[]\n - []\n - []'):
            return {}

        # Prepare context for LLM summary
        context = {
            "concepts": {
                "main_concepts": concept_results.get("concepts", {}),
                "relationships": concept_results.get("relationships", []),
                "concept_groups": concept_results.get("groups", [])
            },
            "topics": [
                {
                    "id": topic["topic_id"],
                    "summary": topic["summary"],
                    "relevance": topic["relevance_score"],
                    "chunk_count": topic["chunk_count"]
                }
                for topic in overview_results.overview
            ],
            "key_chunks": [
                {
                    "text": chunk.text,
                    "metadata": chunk.metadata
                }
                for chunk in basic_results
            ]
        }

        # Generate comprehensive summary using LLM
        system_prompt = """
        Analyze the provided search results and generate a comprehensive summary
        that includes:
        1. Main concepts and their relationships
        2. Key topics and their relevance
        3. Most important findings and insights
        4. Cross-references and connections between topics
        5. Potential gaps or areas for further investigation

        Format the response as a JSON object with these sections.
        """

        prompt = f"""
        Query: {query}

        Context:
        {json.dumps(context, indent=2)}

        Generate a comprehensive analysis and summary following the structure:
        """

        try:
            from toolboxv2.mods.isaa.extras.adapter import litellm_complete
            # await asyncio.sleep(0.25)
            llm_response = await litellm_complete(
                model_name=self.model_name,
                prompt=prompt,
                system_prompt=system_prompt,
                response_format=DataModel,
            )
            summary_analysis = json.loads(llm_response)
        except Exception as e:
            get_logger().error(f"Error generating summary: {str(e)}")
            summary_analysis = {
                "main_summary": "Error generating summary",
                "error": str(e)
            }

        # Compile final results
        return {
            "summary": summary_analysis,
            "raw_results": {
                "concepts": concept_results,
                "overview": {
                    "topics": overview_results.overview,
                    "cross_references": overview_results.cross_references
                },
                "relevant_chunks": [
                    {
                        "text": chunk.text,
                        "metadata": chunk.metadata,
                        "cluster_id": chunk.cluster_id
                    }
                    for chunk in basic_results
                ]
            },
            "metadata": {
                "query": query,
                "timestamp": time.time(),
                "retrieval_params": {
                    "k": k,
                    "min_similarity": min_similarity,
                    "cross_ref_depth": cross_ref_depth,
                    "max_cross_refs": max_cross_refs
                }
            }
        }

    def save(self, path: str) -> bytes | None:
        """
        Save the complete knowledge base to disk, including all sub-components

        Args:
            path (str): Path where the knowledge base will be saved
        """
        try:
            data = {
                # Core components
                'vdb': self.vdb.save(),
                'vis_kwargs': self.vis_kwargs,
                'vis_class': self.vis_class,
                'existing_hashes': self.existing_hashes,

                # Configuration parameters
                'embedding_dim': self.embedding_dim,
                'similarity_threshold': self.similarity_threshold,
                'batch_size': self.batch_size,
                'n_clusters': self.n_clusters,
                'deduplication_threshold': self.deduplication_threshold,
                'model_name': self.model_name,
                'embedding_model': self.embedding_model,

                # Cache and graph data
                'similarity_graph': self.similarity_graph,
                'sto': self.sto,

                # Text splitter configuration
                'text_splitter_config': {
                    'chunk_size': self.text_splitter.chunk_size,
                    'chunk_overlap': self.text_splitter.chunk_overlap,
                    'separator': self.text_splitter.separator
                },

                # Concept extractor data
                'concept_graph': {
                    'concepts': {
                        name: {
                            'name': concept.name,
                            'category': concept.category,
                            'relationships': {k: list(v) for k, v in concept.relationships.items()},
                            'importance_score': concept.importance_score,
                            'context_snippets': concept.context_snippets,
                            'metadata': concept.metadata
                        }
                        for name, concept in self.concept_extractor.concept_graph.concepts.items()
                    }
                }
            }
            b = pickle.dumps(data, protocol=pickle.HIGHEST_PROTOCOL)

            if path is None:
                return b

            path = Path(path)
            tmp = path.with_suffix(path.suffix + ".tmp") if path.suffix else path.with_name(path.name + ".tmp")

            try:
                # Schreibe zuerst in eine temporäre Datei
                with open(tmp, "wb") as f:
                    f.write(b)
                    f.flush()
                    os.fsync(f.fileno())  # sicherstellen, dass die Daten auf Platte sind
                # Atomischer Austausch
                os.replace(tmp, path)
            finally:
                # Aufräumen falls tmp noch existiert (bei Fehlern)
                if tmp.exists():
                    with contextlib.suppress(Exception):
                        tmp.unlink()
            return None
            # print(f"Knowledge base successfully saved to {path} with {len(self.concept_extractor.concept_graph.concepts.items())} concepts")

        except Exception as e:
            print(f"Error saving knowledge base: {str(e)}")
            raise
    def init_vdb(self, db:AbstractVectorStore=AbstractVectorStore):
        pass
    @classmethod
    def load(cls, path: str | bytes) -> 'KnowledgeBase':
        """
        Load a complete knowledge base from disk, including all sub-components

        Args:
            path (str): Path from where to load the knowledge base

        Returns:
            KnowledgeBase: A fully restored knowledge base instance
        """
        try:
            if isinstance(path, bytes | bytearray | memoryview):
                data_bytes = bytes(path)
                try:
                    data = pickle.loads(data_bytes)
                except Exception as e:
                    raise EOFError(f"Fehler beim pickle.loads von bytes: {e}") from e
            else:
                p = Path(path)
                if not p.exists():
                    raise FileNotFoundError(f"{p} existiert nicht")
                size = p.stat().st_size
                if size == 0:
                    raise EOFError(f"{p} ist leer (0 bytes)")
                try:
                    with open(p, "rb") as f:
                        try:
                            data = pickle.load(f)
                        except EOFError as e:
                            # Debug info: erste bytes ausgeben
                            f.seek(0)
                            snippet = f.read(128)
                            raise EOFError(
                                f"EOFError beim Laden {p} (Größe {size} bytes). Erste 128 bytes: {snippet!r}") from e

                except Exception as e:
                    raise ValueError(f"Invalid path type {e}") from e
            # Create new knowledge base instance with saved configuration
            kb = cls(
                embedding_dim=data['embedding_dim'],
                similarity_threshold=data['similarity_threshold'],
                batch_size=data['batch_size'],
                n_clusters=data['n_clusters'],
                deduplication_threshold=data['deduplication_threshold'],
                model_name=data['model_name'],
                embedding_model=data['embedding_model']
            )

            # Restore core components
            kb.init_vis(data.get('vis_class'), data.get('vis_kwargs'))
            kb.existing_hashes = data['existing_hashes']

            # Restore cache and graph data
            kb.similarity_graph = data.get('similarity_graph', {})
            kb.sto = data.get('sto', [])

            # Restore text splitter configuration
            splitter_config = data.get('text_splitter_config', {})
            kb.text_splitter = TextSplitter(
                chunk_size=splitter_config.get('chunk_size', 12_000),
                chunk_overlap=splitter_config.get('chunk_overlap', 200),
                separator=splitter_config.get('separator', '\n')
            )

            # Restore concept graph
            concept_data = data.get('concept_graph', {}).get('concepts', {})
            for concept_info in concept_data.values():
                concept = Concept(
                    name=concept_info['name'],
                    category=concept_info['category'],
                    relationships={k: set(v) for k, v in concept_info['relationships'].items()},
                    importance_score=concept_info['importance_score'],
                    context_snippets=concept_info['context_snippets'],
                    metadata=concept_info['metadata']
                )
                kb.concept_extractor.concept_graph.add_concept(concept)

            # print(f"Knowledge base successfully loaded from {path} with {len(concept_data)} concepts")
            return kb

        except Exception:
            print(f"Error loading knowledge base: {str(e)}")
            import traceback
            traceback.print_exception(e)
            raise

    async def vis(self,output_file: str = "concept_graph.html", get_output_html=False, get_output_net=False):

        if not self.concept_extractor.concept_graph.concepts:

            if len(self.sto) > 2:
                await self.add_data([t for (t, m) in self.sto], [m for (t, m) in self.sto], direct=True)
                # self.sto = []
            if not self.concept_extractor.concept_graph.concepts:
                print("NO Concepts defined and no data in sto")
                return None


        net = self.concept_extractor.concept_graph.convert_to_networkx()
        if get_output_net:
            return net
        return GraphVisualizer.visualize(net, output_file=output_file, get_output=get_output_html)

__init__(embedding_dim=256, similarity_threshold=0.61, batch_size=12, n_clusters=4, deduplication_threshold=0.85, model_name=os.getenv('SUMMARYMODEL'), embedding_model=os.getenv('DEFAULTMODELEMBEDDING'), vis_class='FaissVectorStore', vis_kwargs=None, requests_per_second=85.0, chunk_size=3600, chunk_overlap=130, separator='\n')

Initialize the knowledge base with given parameters

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py

def __init__(self, embedding_dim: int = 256, similarity_threshold: float = 0.61, batch_size: int = 12,
             n_clusters: int = 4, deduplication_threshold: float = 0.85, model_name=os.getenv("SUMMARYMODEL"),
             embedding_model=os.getenv("DEFAULTMODELEMBEDDING"),
             vis_class:str | None = "FaissVectorStore",
             vis_kwargs:dict[str, Any] | None=None,
             requests_per_second=85.,
             chunk_size: int = 3600,
             chunk_overlap: int = 130,
             separator: str = "\n"
             ):
    """Initialize the knowledge base with given parameters"""

    self.existing_hashes: set[str] = set()
    self.embedding_model = embedding_model
    self.embedding_dim = embedding_dim
    self.similarity_threshold = similarity_threshold
    self.deduplication_threshold = deduplication_threshold
    if model_name == "openrouter/mistralai/mistral-nemo":
        batch_size = 9
        requests_per_second = 1.5
    self.batch_size = batch_size
    self.n_clusters = n_clusters
    self.model_name = model_name
    self.sto: list = []

    self.text_splitter = TextSplitter(chunk_size=chunk_size,chunk_overlap=chunk_overlap, separator=separator)
    self.similarity_graph = {}
    self.concept_extractor = ConceptExtractor(self, requests_per_second)

    self.vis_class = None
    self.vis_kwargs = None
    self.vdb = None
    self.init_vis(vis_class, vis_kwargs)

add_data(texts, metadata=None, direct=False) async

Enhanced version with smart splitting and clustering

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py

async def add_data(
    self,
    texts: list[str],
    metadata: list[dict[str, Any]] | None = None, direct:bool = False
) -> tuple[int, int]:
    """Enhanced version with smart splitting and clustering"""
    if isinstance(texts, str):
        texts = [texts]
    if metadata is None:
        metadata = [{}] * len(texts)
    if isinstance(metadata, dict):
        metadata = [metadata]
    if len(texts) != len(metadata):
        raise ValueError("Length of texts and metadata must match")

    if not direct and len(texts) == 1 and len(texts[0]) < 10_000:
        if len(self.sto) < self.batch_size and len(texts) == 1:
            self.sto.append((texts[0], metadata[0]))
            return -1, -1
        if len(self.sto) >= self.batch_size:
            _ = [texts.append(t) or metadata.append([m]) for (t, m) in self.sto]
            self.sto = []

    # Split large texts
    split_texts = []
    split_metadata = []

    while Spinner("Saving Data to Memory", symbols='t'):

        for idx, text in enumerate(texts):
            chunks = self.text_splitter.split_text(text)
            split_texts.extend(chunks)

            # Adjust metadata for splits
            meta = metadata[idx] if metadata else {}
            if isinstance(meta, list):
                meta = meta[0]
            for i, _chunk in enumerate(chunks):
                chunk_meta = meta.copy()
                chunk_meta.update({
                    'chunk_index': i,
                    'total_chunks': len(chunks),
                    'original_text_id': idx
                })
                split_metadata.append(chunk_meta)

        return await self._add_data(split_texts, split_metadata)

compute_hash(text) staticmethod

Compute SHA-256 hash of text

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py

@staticmethod
def compute_hash(text: str) -> str:
    """Compute SHA-256 hash of text"""
    return hashlib.sha256(text.encode('utf-8', errors='ignore')).hexdigest()

forget_irrelevant(irrelevant_concepts, similarity_threshold=None) async

Remove chunks similar to irrelevant concepts Returns: Number of chunks removed

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py

async def forget_irrelevant(self, irrelevant_concepts: list[str], similarity_threshold: float | None=None) -> int:
    """
    Remove chunks similar to irrelevant concepts
    Returns: Number of chunks removed
    """
    if not irrelevant_concepts:
        return 0

    if similarity_threshold is None:
        similarity_threshold = self.similarity_threshold

    try:
        irrelevant_embeddings = await self._get_embeddings(irrelevant_concepts)
        initial_count = len(self.vdb.chunks)

        def is_relevant(chunk: Chunk) -> bool:
            similarities = np.dot(chunk.embedding, irrelevant_embeddings.T)
            do_keep = np.max(similarities) < similarity_threshold
            if do_keep:
                return True
            for c in chunk.metadata.get("concepts", []):
                if c in self.concept_extractor.concept_graph.concepts:
                    del self.concept_extractor.concept_graph.concepts[c]
            return False

        relevant_chunks = [chunk for chunk in self.vdb.chunks if is_relevant(chunk)]
        self.vdb.chunks = relevant_chunks
        self.existing_hashes = {chunk.content_hash for chunk in self.vdb.chunks}
        self.vdb.rebuild_index()

        return initial_count - len(self.vdb.chunks)

    except Exception as e:
        get_logger().error(f"Error forgetting irrelevant concepts: {str(e)}")
        raise

load(path) classmethod

Load a complete knowledge base from disk, including all sub-components

Parameters:

Name	Type	Description	Default
path	str	Path from where to load the knowledge base	required

Returns:

Name	Type	Description
KnowledgeBase	KnowledgeBase	A fully restored knowledge base instance

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py

@classmethod
def load(cls, path: str | bytes) -> 'KnowledgeBase':
    """
    Load a complete knowledge base from disk, including all sub-components

    Args:
        path (str): Path from where to load the knowledge base

    Returns:
        KnowledgeBase: A fully restored knowledge base instance
    """
    try:
        if isinstance(path, bytes | bytearray | memoryview):
            data_bytes = bytes(path)
            try:
                data = pickle.loads(data_bytes)
            except Exception as e:
                raise EOFError(f"Fehler beim pickle.loads von bytes: {e}") from e
        else:
            p = Path(path)
            if not p.exists():
                raise FileNotFoundError(f"{p} existiert nicht")
            size = p.stat().st_size
            if size == 0:
                raise EOFError(f"{p} ist leer (0 bytes)")
            try:
                with open(p, "rb") as f:
                    try:
                        data = pickle.load(f)
                    except EOFError as e:
                        # Debug info: erste bytes ausgeben
                        f.seek(0)
                        snippet = f.read(128)
                        raise EOFError(
                            f"EOFError beim Laden {p} (Größe {size} bytes). Erste 128 bytes: {snippet!r}") from e

            except Exception as e:
                raise ValueError(f"Invalid path type {e}") from e
        # Create new knowledge base instance with saved configuration
        kb = cls(
            embedding_dim=data['embedding_dim'],
            similarity_threshold=data['similarity_threshold'],
            batch_size=data['batch_size'],
            n_clusters=data['n_clusters'],
            deduplication_threshold=data['deduplication_threshold'],
            model_name=data['model_name'],
            embedding_model=data['embedding_model']
        )

        # Restore core components
        kb.init_vis(data.get('vis_class'), data.get('vis_kwargs'))
        kb.existing_hashes = data['existing_hashes']

        # Restore cache and graph data
        kb.similarity_graph = data.get('similarity_graph', {})
        kb.sto = data.get('sto', [])

        # Restore text splitter configuration
        splitter_config = data.get('text_splitter_config', {})
        kb.text_splitter = TextSplitter(
            chunk_size=splitter_config.get('chunk_size', 12_000),
            chunk_overlap=splitter_config.get('chunk_overlap', 200),
            separator=splitter_config.get('separator', '\n')
        )

        # Restore concept graph
        concept_data = data.get('concept_graph', {}).get('concepts', {})
        for concept_info in concept_data.values():
            concept = Concept(
                name=concept_info['name'],
                category=concept_info['category'],
                relationships={k: set(v) for k, v in concept_info['relationships'].items()},
                importance_score=concept_info['importance_score'],
                context_snippets=concept_info['context_snippets'],
                metadata=concept_info['metadata']
            )
            kb.concept_extractor.concept_graph.add_concept(concept)

        # print(f"Knowledge base successfully loaded from {path} with {len(concept_data)} concepts")
        return kb

    except Exception:
        print(f"Error loading knowledge base: {str(e)}")
        import traceback
        traceback.print_exception(e)
        raise

query_concepts(query) async

Query concepts extracted from the knowledge base

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py

async def query_concepts(self, query: str) -> dict[str, any]:
    """Query concepts extracted from the knowledge base"""
    return await self.concept_extractor.query_concepts(query)

retrieve(query='', query_embedding=None, k=5, min_similarity=0.2, include_connected=True) async

Enhanced retrieval with connected information

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py

async def retrieve(
    self,
    query: str="",
    query_embedding: np.ndarray | None = None,
    k: int = 5,
    min_similarity: float = 0.2,
    include_connected: bool = True
) -> list[Chunk]:
    """Enhanced retrieval with connected information"""
    if query_embedding is None:
        query_embedding = (await self._get_embeddings([query]))[0]
    k = min(k, len(self.vdb.chunks))
    if k <= 0:
        return []
    initial_results = self.vdb.search(query_embedding, k, min_similarity)

    if not include_connected or not initial_results:
        return initial_results

    # Find connected chunks
    connected_chunks = set()
    for chunk in initial_results:
        chunk_id = self.vdb.chunks.index(chunk)
        if chunk_id in self.similarity_graph:
            connected_chunks.update(self.similarity_graph[chunk_id])

    # Add connected chunks to results
    all_chunks = self.vdb.chunks
    additional_results = [all_chunks[i] for i in connected_chunks
                          if all_chunks[i] not in initial_results]

    # Sort by similarity to query
    all_results = initial_results + additional_results

    return sorted(
        all_results,
        key=lambda x: np.dot(x.embedding, query_embedding),
        reverse=True
    )[:k * 2]  # Return more results when including connected information

retrieve_with_overview(query, query_embedding=None, k=5, min_similarity=0.2, max_sentences=5, cross_ref_depth=2, max_cross_refs=10) async

Enhanced retrieval with better cross-reference handling

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py

async def retrieve_with_overview(
    self,
    query: str,
    query_embedding=None,
    k: int = 5,
    min_similarity: float = 0.2,
    max_sentences: int = 5,
    cross_ref_depth: int = 2,
    max_cross_refs: int = 10  # New parameter to control cross-reference count
) -> RetrievalResult:
    """Enhanced retrieval with better cross-reference handling"""
    # Get initial results with query embedding
    if query_embedding is None:
        query_embedding = (await self._get_embeddings([query]))[0]
    initial_results = await self.retrieve(query_embedding=query_embedding, k=k, min_similarity=min_similarity)

    if not initial_results:
        return RetrievalResult([], [], {})

    # Find cross-references with similarity scoring
    initial_ids = {self.vdb.chunks.index(chunk) for chunk in initial_results}
    related_ids = self._find_cross_references(
        initial_ids,
        depth=cross_ref_depth,
        query_embedding=query_embedding  # Pass query embedding for relevance scoring
    )

    # Get all relevant chunks with smarter filtering
    all_chunks = self.vdb.chunks
    all_relevant_chunks = initial_results + [
        chunk for i, chunk in enumerate(all_chunks)
        if i in related_ids and self._is_relevant_cross_ref(
            chunk,
            query_embedding,
            initial_results
        )
    ]

    # Enhanced clustering with dynamic cluster size
    clusters = self._cluster_chunks(
        all_relevant_chunks,
        query_embedding=query_embedding
    )

    # Fallback: If no clusters are found, treat all relevant chunks as a single cluster.
    if not clusters:
        print("No clusters found. Falling back to using all relevant chunks as a single cluster.")
        clusters = {0: all_relevant_chunks}

    # Generate summaries and organize results
    overview = []
    cross_references = {}

    for cluster_id, cluster_chunks in clusters.items():
        summary = self._generate_topic_summary(
            cluster_chunks,
            query_embedding,
            max_sentences=max_sentences  # Increased for more context
        )

        # Enhanced chunk sorting with combined scoring
        sorted_chunks = self._sort_chunks_by_relevance(
            cluster_chunks,
            query_embedding,
            initial_results
        )

        # Separate direct matches and cross-references
        direct_matches_ = [{'text':c.text, 'metadata':c.metadata} for c in sorted_chunks if c in initial_results]
        direct_matches = []
        for match in direct_matches_:
            if match in direct_matches:
                continue
            direct_matches.append(match)
        cross_refs_ = [c for c in sorted_chunks if c not in initial_results]
        cross_refs = []
        for match in cross_refs_:
            if match in cross_refs:
                continue
            cross_refs.append(match)
        # Limit cross-references while maintaining diversity
        selected_cross_refs = self._select_diverse_cross_refs(
            cross_refs,
            max_cross_refs,
            query_embedding
        )

        topic_info = {
            'topic_id': cluster_id,
            'summary': summary,
            'main_chunks': [x for x in direct_matches[:3]],
            'chunk_count': len(cluster_chunks),
            'relevance_score': self._calculate_topic_relevance(
                cluster_chunks,
                query_embedding
            )
        }
        overview.append(topic_info)

        if selected_cross_refs:
            cross_references[f"topic_{cluster_id}"] = selected_cross_refs

    # Sort overview by relevance score
    overview.sort(key=lambda x: x['relevance_score'], reverse=True)

    return RetrievalResult(
        overview=overview,
        details=initial_results,
        cross_references=cross_references
    )

save(path)

Save the complete knowledge base to disk, including all sub-components

Parameters:

Name	Type	Description	Default
path	str	Path where the knowledge base will be saved	required

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py

def save(self, path: str) -> bytes | None:
    """
    Save the complete knowledge base to disk, including all sub-components

    Args:
        path (str): Path where the knowledge base will be saved
    """
    try:
        data = {
            # Core components
            'vdb': self.vdb.save(),
            'vis_kwargs': self.vis_kwargs,
            'vis_class': self.vis_class,
            'existing_hashes': self.existing_hashes,

            # Configuration parameters
            'embedding_dim': self.embedding_dim,
            'similarity_threshold': self.similarity_threshold,
            'batch_size': self.batch_size,
            'n_clusters': self.n_clusters,
            'deduplication_threshold': self.deduplication_threshold,
            'model_name': self.model_name,
            'embedding_model': self.embedding_model,

            # Cache and graph data
            'similarity_graph': self.similarity_graph,
            'sto': self.sto,

            # Text splitter configuration
            'text_splitter_config': {
                'chunk_size': self.text_splitter.chunk_size,
                'chunk_overlap': self.text_splitter.chunk_overlap,
                'separator': self.text_splitter.separator
            },

            # Concept extractor data
            'concept_graph': {
                'concepts': {
                    name: {
                        'name': concept.name,
                        'category': concept.category,
                        'relationships': {k: list(v) for k, v in concept.relationships.items()},
                        'importance_score': concept.importance_score,
                        'context_snippets': concept.context_snippets,
                        'metadata': concept.metadata
                    }
                    for name, concept in self.concept_extractor.concept_graph.concepts.items()
                }
            }
        }
        b = pickle.dumps(data, protocol=pickle.HIGHEST_PROTOCOL)

        if path is None:
            return b

        path = Path(path)
        tmp = path.with_suffix(path.suffix + ".tmp") if path.suffix else path.with_name(path.name + ".tmp")

        try:
            # Schreibe zuerst in eine temporäre Datei
            with open(tmp, "wb") as f:
                f.write(b)
                f.flush()
                os.fsync(f.fileno())  # sicherstellen, dass die Daten auf Platte sind
            # Atomischer Austausch
            os.replace(tmp, path)
        finally:
            # Aufräumen falls tmp noch existiert (bei Fehlern)
            if tmp.exists():
                with contextlib.suppress(Exception):
                    tmp.unlink()
        return None
        # print(f"Knowledge base successfully saved to {path} with {len(self.concept_extractor.concept_graph.concepts.items())} concepts")

    except Exception as e:
        print(f"Error saving knowledge base: {str(e)}")
        raise

unified_retrieve(query, k=5, min_similarity=0.2, cross_ref_depth=2, max_cross_refs=10, max_sentences=10) async

Unified retrieval function that combines concept querying, retrieval with overview, and basic retrieval, then generates a comprehensive summary using LLM.

Parameters:

Name	Type	Description	Default
query	str	Search query string	required
k	int	Number of primary results to retrieve	5
min_similarity	float	Minimum similarity threshold for retrieval	0.2
cross_ref_depth	int	Depth for cross-reference search	2
max_cross_refs	int	Maximum number of cross-references per topic	10
max_sentences	int	Maximum number Sentences in the main summary text	10

Returns:

Type	Description
dict[str, Any]	Dictionary containing comprehensive results including summary and details

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py

async def unified_retrieve(
    self,
    query: str,
    k: int = 5,
    min_similarity: float = 0.2,
    cross_ref_depth: int = 2,
    max_cross_refs: int = 10,
    max_sentences: int = 10
) -> dict[str, Any]:
    """
    Unified retrieval function that combines concept querying, retrieval with overview,
    and basic retrieval, then generates a comprehensive summary using LLM.

    Args:
        query: Search query string
        k: Number of primary results to retrieve
        min_similarity: Minimum similarity threshold for retrieval
        cross_ref_depth: Depth for cross-reference search
        max_cross_refs: Maximum number of cross-references per topic
        max_sentences: Maximum number Sentences in the main summary text

    Returns:
        Dictionary containing comprehensive results including summary and details
    """
    # Get concept information
    concept_results = await self.concept_extractor.query_concepts(query)

    # Get retrieval overview

    query_embedding = (await self._get_embeddings([query]))[0]
    overview_results = await self.retrieve_with_overview(
        query=query,
        query_embedding=query_embedding,
        k=k,
        min_similarity=min_similarity,
        cross_ref_depth=cross_ref_depth,
        max_cross_refs=max_cross_refs,
        max_sentences=max_sentences
    )

    # Get basic retrieval results
    basic_results = await self.retrieve(
        query_embedding=query_embedding,
        k=k,
        min_similarity=min_similarity
    )
    if len(basic_results) == 0:
        return {}
    if len(basic_results) == 1 and isinstance(basic_results[0], str) and basic_results[0].endswith('[]\n - []\n - []'):
        return {}

    # Prepare context for LLM summary
    context = {
        "concepts": {
            "main_concepts": concept_results.get("concepts", {}),
            "relationships": concept_results.get("relationships", []),
            "concept_groups": concept_results.get("groups", [])
        },
        "topics": [
            {
                "id": topic["topic_id"],
                "summary": topic["summary"],
                "relevance": topic["relevance_score"],
                "chunk_count": topic["chunk_count"]
            }
            for topic in overview_results.overview
        ],
        "key_chunks": [
            {
                "text": chunk.text,
                "metadata": chunk.metadata
            }
            for chunk in basic_results
        ]
    }

    # Generate comprehensive summary using LLM
    system_prompt = """
    Analyze the provided search results and generate a comprehensive summary
    that includes:
    1. Main concepts and their relationships
    2. Key topics and their relevance
    3. Most important findings and insights
    4. Cross-references and connections between topics
    5. Potential gaps or areas for further investigation

    Format the response as a JSON object with these sections.
    """

    prompt = f"""
    Query: {query}

    Context:
    {json.dumps(context, indent=2)}

    Generate a comprehensive analysis and summary following the structure:
    """

    try:
        from toolboxv2.mods.isaa.extras.adapter import litellm_complete
        # await asyncio.sleep(0.25)
        llm_response = await litellm_complete(
            model_name=self.model_name,
            prompt=prompt,
            system_prompt=system_prompt,
            response_format=DataModel,
        )
        summary_analysis = json.loads(llm_response)
    except Exception as e:
        get_logger().error(f"Error generating summary: {str(e)}")
        summary_analysis = {
            "main_summary": "Error generating summary",
            "error": str(e)
        }

    # Compile final results
    return {
        "summary": summary_analysis,
        "raw_results": {
            "concepts": concept_results,
            "overview": {
                "topics": overview_results.overview,
                "cross_references": overview_results.cross_references
            },
            "relevant_chunks": [
                {
                    "text": chunk.text,
                    "metadata": chunk.metadata,
                    "cluster_id": chunk.cluster_id
                }
                for chunk in basic_results
            ]
        },
        "metadata": {
            "query": query,
            "timestamp": time.time(),
            "retrieval_params": {
                "k": k,
                "min_similarity": min_similarity,
                "cross_ref_depth": cross_ref_depth,
                "max_cross_refs": max_cross_refs
            }
        }
    }

RelevanceAssessment

Bases: BaseModel

Represents an assessment of the relevance of the data in relation to a specific query.

Attributes:

Name	Type	Description
query_alignment	float	A float representing the alignment between the query and the data.
confidence_score	float	A float indicating the confidence level in the alignment.
coverage_analysis	str	A textual description analyzing the data coverage.

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py

class RelevanceAssessment(BaseModel):
    """
    Represents an assessment of the relevance of the data in relation to a specific query.

    Attributes:
        query_alignment (float): A float representing the alignment between the query and the data.
        confidence_score (float): A float indicating the confidence level in the alignment.
        coverage_analysis (str): A textual description analyzing the data coverage.
    """
    query_alignment: float
    confidence_score: float
    coverage_analysis: str

RetrievalResult dataclass

Structure for organizing retrieval results

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py

@dataclass
class RetrievalResult:
    """Structure for organizing retrieval results"""
    overview: list[dict[str, any]]  # List of topic summaries
    details: list[Chunk]  # Detailed chunks
    cross_references: dict[str, list[Chunk]]  # Related chunks by topic

TConcept

Bases: BaseModel

Represents the criteria or target parameters for concept selection and filtering.

Attributes:

Name	Type	Description
min_importance	float	The minimum importance score a concept must have to be considered.
target_concepts	List[str]	A list of names of target concepts to focus on.
relationship_types	List[str]	A list of relationship types to be considered in the analysis.
categories	List[str]	A list of concept categories to filter or group the concepts.

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py

class TConcept(BaseModel):
    """
    Represents the criteria or target parameters for concept selection and filtering.

    Attributes:
        min_importance (float): The minimum importance score a concept must have to be considered.
        target_concepts (List[str]): A list of names of target concepts to focus on.
        relationship_types (List[str]): A list of relationship types to be considered in the analysis.
        categories (List[str]): A list of concept categories to filter or group the concepts.
    """
    min_importance: float
    target_concepts: list[str]
    relationship_types: list[str]
    categories: list[str]

TextSplitter

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py

class TextSplitter:
    def __init__(
        self,
        chunk_size: int = 3600,
        chunk_overlap: int = 130,
        separator: str = "\n"
    ):
        self.chunk_size = chunk_size
        self.chunk_overlap = chunk_overlap
        self.separator = separator

    def approximate(self, text_len: int) -> float:
        """
        Approximate the number of chunks and average chunk size for a given text length

        Args:
            text_len (int): Length of the text to be split

        Returns:
            Tuple[int, int]: (number_of_chunks, approximate_chunk_size)
        """
        if text_len <= self.chunk_size:
            return 1, text_len

        # Handle extreme overlap cases
        if self.chunk_overlap >= self.chunk_size:
            estimated_chunks = text_len
            return estimated_chunks, 1

        # Calculate based on overlap ratio
        overlap_ratio = self.chunk_overlap / self.chunk_size
        base_chunks = text_len / self.chunk_size
        estimated_chunks = base_chunks * 2 / (overlap_ratio if overlap_ratio > 0 else 1)

        # print('#',estimated_chunks, base_chunks, overlap_ratio)
        # Calculate average chunk size
        avg_chunk_size = max(1, text_len / estimated_chunks)

        return estimated_chunks * avg_chunk_size

    def split_text(self, text: str) -> list[str]:
        """Split text into chunks with overlap"""
        # Clean and normalize text
        text = re.sub(r'\s+', ' ', text).strip()

        # If text is shorter than chunk_size, return as is
        if len(text) <= self.chunk_size:
            return [text]

        chunks = []
        start = 0

        while start < len(text):
            # Find end of chunk
            end = start + self.chunk_size

            if end >= len(text):
                chunks.append(text[start:])
                break

            # Try to find a natural break point
            last_separator = text.rfind(self.separator, start, end)
            if last_separator != -1:
                end = last_separator

            # Add chunk
            chunks.append(text[start:end])

            # Calculate allowed overlap for this chunk
            chunk_length = end - start
            allowed_overlap = min(self.chunk_overlap, chunk_length - 1)

            # Move start position considering adjusted overlap
            start = end - allowed_overlap

        return chunks

approximate(text_len)

Approximate the number of chunks and average chunk size for a given text length

Parameters:

Name	Type	Description	Default
text_len	int	Length of the text to be split	required

Returns:

Type	Description
float	Tuple[int, int]: (number_of_chunks, approximate_chunk_size)

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py

def approximate(self, text_len: int) -> float:
    """
    Approximate the number of chunks and average chunk size for a given text length

    Args:
        text_len (int): Length of the text to be split

    Returns:
        Tuple[int, int]: (number_of_chunks, approximate_chunk_size)
    """
    if text_len <= self.chunk_size:
        return 1, text_len

    # Handle extreme overlap cases
    if self.chunk_overlap >= self.chunk_size:
        estimated_chunks = text_len
        return estimated_chunks, 1

    # Calculate based on overlap ratio
    overlap_ratio = self.chunk_overlap / self.chunk_size
    base_chunks = text_len / self.chunk_size
    estimated_chunks = base_chunks * 2 / (overlap_ratio if overlap_ratio > 0 else 1)

    # print('#',estimated_chunks, base_chunks, overlap_ratio)
    # Calculate average chunk size
    avg_chunk_size = max(1, text_len / estimated_chunks)

    return estimated_chunks * avg_chunk_size

split_text(text)

Split text into chunks with overlap

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py

def split_text(self, text: str) -> list[str]:
    """Split text into chunks with overlap"""
    # Clean and normalize text
    text = re.sub(r'\s+', ' ', text).strip()

    # If text is shorter than chunk_size, return as is
    if len(text) <= self.chunk_size:
        return [text]

    chunks = []
    start = 0

    while start < len(text):
        # Find end of chunk
        end = start + self.chunk_size

        if end >= len(text):
            chunks.append(text[start:])
            break

        # Try to find a natural break point
        last_separator = text.rfind(self.separator, start, end)
        if last_separator != -1:
            end = last_separator

        # Add chunk
        chunks.append(text[start:end])

        # Calculate allowed overlap for this chunk
        chunk_length = end - start
        allowed_overlap = min(self.chunk_overlap, chunk_length - 1)

        # Move start position considering adjusted overlap
        start = end - allowed_overlap

    return chunks

TopicInsights

Bases: BaseModel

Represents insights related to various topics.

Attributes:

Name	Type	Description
primary_topics	list[str]	A list of main topics addressed.
cross_references	list[str]	A list of cross-references that connect different topics.
knowledge_gaps	list[str]	A list of identified gaps in the current knowledge.

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py

class TopicInsights(BaseModel):
    """
    Represents insights related to various topics.

    Attributes:
        primary_topics (list[str]): A list of main topics addressed.
        cross_references (list[str]): A list of cross-references that connect different topics.
        knowledge_gaps (list[str]): A list of identified gaps in the current knowledge.
    """
    primary_topics: list[str]
    cross_references: list[str]
    knowledge_gaps: list[str]

rConcept

Bases: BaseModel

Represents a key concept with its relationships and associated metadata.

Attributes:

Name	Type	Description
name	str	The name of the concept.
category	str	The category of the concept (e.g., 'technical', 'domain', 'method', etc.).
relationships	Dict[str, List[str]]	A mapping where each key is a type of relationship and the value is a list of related concept names.
importance_score	float	A numerical score representing the importance or relevance of the concept.
context_snippets	List[str]	A list of text snippets providing context where the concept appears.

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py

class rConcept(BaseModel):
    """
    Represents a key concept with its relationships and associated metadata.

    Attributes:
        name (str): The name of the concept.
        category (str): The category of the concept (e.g., 'technical', 'domain', 'method', etc.).
        relationships (Dict[str, List[str]]): A mapping where each key is a type of relationship and the
            value is a list of related concept names.
        importance_score (float): A numerical score representing the importance or relevance of the concept.
        context_snippets (List[str]): A list of text snippets providing context where the concept appears.
    """
    name: str
    category: str
    relationships: dict[str, list[str]]
    importance_score: float
    context_snippets: list[str]

normalize_vectors(vectors)

Normalize vectors to unit length

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py

def normalize_vectors(vectors: np.ndarray) -> np.ndarray:
    """Normalize vectors to unit length"""
    norms = np.linalg.norm(vectors, axis=1, keepdims=True)
    return np.divide(vectors, norms, where=norms != 0)

VectorStores

Vector store implementations for the toolboxv2 system.

taichiNumpyNumbaVectorStores

NumpyVectorStore

Bases: AbstractVectorStore

Source code in toolboxv2/mods/isaa/base/VectorStores/taichiNumpyNumbaVectorStores.py

class NumpyVectorStore(AbstractVectorStore):
    def __init__(self, use_gpu=False):
        self.embeddings = np.empty((0, 0))
        self.chunks = []
        # Initialize Taich


        self.normalized_embeddings = None

    def add_embeddings(self, embeddings: np.ndarray, chunks: list[Chunk]) -> None:
        if len(embeddings.shape) != 2:
            raise ValueError("Embeddings must be 2D array")
        if len(chunks) != embeddings.shape[0]:
            raise ValueError("Mismatch between embeddings and chunks count")

        if self.embeddings.size == 0:
            self.embeddings = embeddings
        else:
            if embeddings.shape[1] != self.embeddings.shape[1]:
                raise ValueError("Embedding dimensions must match")
            self.embeddings = np.vstack([self.embeddings, embeddings])
        self.chunks.extend(chunks)
        # Reset normalized embeddings cache
        self.normalized_embeddings = None

    def search(self, query_embedding: np.ndarray, k: int = 5, min_similarity: float = 0.7) -> list[Chunk]:
        if self.embeddings.size == 0:
            return []

        # Pre-compute normalized embeddings if not cached
        if self.normalized_embeddings is None:
            self._precompute_normalized_embeddings()

        # Normalize query
        query_norm = self._normalize_vector(query_embedding)

        # Enhanced Taichi kernel for similarity computation
        n = len(self.chunks)
        similarities = np.zeros(n, dtype=np.float32)

        @ti.kernel
        def compute_similarities_optimized(
            query: ti.types.ndarray(dtype=ti.f32),
            embeddings: ti.types.ndarray(dtype=ti.f32),
            similarities: ti.types.ndarray(dtype=ti.f32),
            n: ti.i32,
            dim: ti.i32
        ):
            ti.loop_config(block_dim=256)
            for i in range(n):
                dot_product = 0.0
                # Vectorized dot product computation
                for j in range(dim):
                    dot_product += embeddings[i, j] * query[j]
                similarities[i] = dot_product

        # Alternative optimized kernel using tile-based computation
        @ti.kernel
        def compute_similarities_tiled(
            query: ti.types.ndarray(dtype=ti.f32),
            embeddings: ti.types.ndarray(dtype=ti.f32),
            similarities: ti.types.ndarray(dtype=ti.f32),
            n: ti.i32,
            dim: ti.i32
        ):
            tile_size = 16  # Adjust based on hardware
            for i in range(n):
                dot_product = 0.0
                # Process in tiles for better cache utilization
                for jt in range(0, dim):
                    if jt % tile_size != 0:
                        continue
                    tile_sum = 0.0
                    for j in range(jt, ti.min(jt + tile_size, dim)):
                        tile_sum += embeddings[i, j] * query[j]
                    dot_product += tile_sum
                similarities[i] = dot_product

        # Choose the appropriate kernel based on dimension size
        if query_embedding.shape[0] >= 256:
            compute_similarities_tiled(
                query_norm.astype(np.float32),
                self.normalized_embeddings,
                similarities,
                n,
                query_embedding.shape[0]
            )
        else:
            compute_similarities_optimized(
                query_norm.astype(np.float32),
                self.normalized_embeddings,
                similarities,
                n,
                query_embedding.shape[0]
            )

        # Optimize top-k selection
        if k >= n:
            indices = np.argsort(-similarities)
        else:
            # Use partial sort for better performance when k < n
            indices = np.argpartition(-similarities, k)[:k]
            indices = indices[np.argsort(-similarities[indices])]

        # Filter results efficiently using vectorized operations
        mask = similarities[indices] >= min_similarity
        filtered_indices = indices[mask]
        return [self.chunks[idx] for idx in filtered_indices[:k]]

    def save(self) -> bytes:
        return pickle.dumps({
            'embeddings': self.embeddings,
            'chunks': self.chunks
        })

    def load(self, data: bytes) -> 'NumpyVectorStore':
        loaded = pickle.loads(data)
        self.embeddings = loaded['embeddings']
        self.chunks = loaded['chunks']
        return self

    def clear(self) -> None:
        self.embeddings = np.empty((0, 0))
        self.chunks = []
        self.normalized_embeddings = None

    def rebuild_index(self) -> None:
        pass  # No index to rebuild for numpy implementation

    def _normalize_vector(self, vector: np.ndarray) -> np.ndarray:
        """Normalize a single vector efficiently."""
        return vector / (np.linalg.norm(vector) + 1e-8)

    def _precompute_normalized_embeddings(self) -> None:
        """Pre-compute and cache normalized embeddings."""
        # Allocate output array
        self.normalized_embeddings = np.empty_like(self.embeddings, dtype=np.float32)

        # Normalize embeddings using Taichi
        batch_normalize(
            self.embeddings.astype(np.float32),
            self.normalized_embeddings,
            self.embeddings.shape[0],
            self.embeddings.shape[1]
        )

types

AbstractVectorStore

Bases: ABC

Abstract base class for vector stores

Source code in toolboxv2/mods/isaa/base/VectorStores/types.py

class AbstractVectorStore(ABC):
    """Abstract base class for vector stores"""

    @abstractmethod
    def add_embeddings(self, embeddings: np.ndarray, chunks: list[Chunk]) -> None:
        """Add embeddings and their corresponding chunks to the store"""
        pass

    @abstractmethod
    def search(self, query_embedding: np.ndarray, k: int = 5, min_similarity: float = 0.7) -> list[Chunk]:
        """Search for similar vectors"""
        pass

    @abstractmethod
    def save(self) -> bytes:
        """Save the vector store to disk"""
        pass

    @abstractmethod
    def load(self, data: bytes) -> 'AbstractVectorStore':
        """Load the vector store from disk"""
        pass

    @abstractmethod
    def clear(self) -> None:
        """Clear all data from the store"""
        pass

    @abstractmethod
    def rebuild_index(self) -> None:
        """Optional for faster searches"""
        pass

add_embeddings(embeddings, chunks) abstractmethod

Add embeddings and their corresponding chunks to the store

Source code in toolboxv2/mods/isaa/base/VectorStores/types.py

@abstractmethod
def add_embeddings(self, embeddings: np.ndarray, chunks: list[Chunk]) -> None:
    """Add embeddings and their corresponding chunks to the store"""
    pass

clear() abstractmethod

Clear all data from the store

Source code in toolboxv2/mods/isaa/base/VectorStores/types.py

@abstractmethod
def clear(self) -> None:
    """Clear all data from the store"""
    pass

load(data) abstractmethod

Load the vector store from disk

Source code in toolboxv2/mods/isaa/base/VectorStores/types.py

@abstractmethod
def load(self, data: bytes) -> 'AbstractVectorStore':
    """Load the vector store from disk"""
    pass

rebuild_index() abstractmethod

Optional for faster searches

Source code in toolboxv2/mods/isaa/base/VectorStores/types.py

@abstractmethod
def rebuild_index(self) -> None:
    """Optional for faster searches"""
    pass

save() abstractmethod

Save the vector store to disk

Source code in toolboxv2/mods/isaa/base/VectorStores/types.py

@abstractmethod
def save(self) -> bytes:
    """Save the vector store to disk"""
    pass

search(query_embedding, k=5, min_similarity=0.7) abstractmethod

Search for similar vectors

Source code in toolboxv2/mods/isaa/base/VectorStores/types.py

@abstractmethod
def search(self, query_embedding: np.ndarray, k: int = 5, min_similarity: float = 0.7) -> list[Chunk]:
    """Search for similar vectors"""
    pass

Chunk dataclass

Represents a chunk of text with its embedding and metadata

Source code in toolboxv2/mods/isaa/base/VectorStores/types.py

@dataclass(slots=True)
class Chunk:
    """Represents a chunk of text with its embedding and metadata"""
    text: str
    embedding: np.ndarray
    metadata: dict[str, Any]
    content_hash: str
    cluster_id: int | None = None

extras

adapter

LiteLLM LLM Interface Module

This module provides interfaces for interacting with LiteLLM's language models, including text generation and embedding capabilities.

Author: Lightrag Team Created: 2025-02-04 License: MIT License Version: 1.0.0

Change Log: - 1.0.0 (2025-02-04): Initial LiteLLM release * Ported OpenAI logic to use litellm async client * Updated error types and environment variable names * Preserved streaming and embedding support

Dependencies

• litellm
• numpy
• pipmaster
• Python >= 3.10

Usage

from llm_interfaces.litellm import logging

if not hasattr(logging, 'NONE'): logging.NONE = 100

import litellm_complete, litellm_embed

litellm_complete(prompt, system_prompt=None, history_messages=None, keyword_extraction=False, model_name='groq/gemma2-9b-it', **kwargs) async

Public completion interface using the model name specified in the global configuration. Optionally extracts keywords if requested.

Source code in toolboxv2/mods/isaa/extras/adapter.py

async def litellm_complete(
    prompt, system_prompt=None, history_messages=None, keyword_extraction=False, model_name = "groq/gemma2-9b-it", **kwargs
) -> str | AsyncIterator[str]:
    """
    Public completion interface using the model name specified in the global configuration.
    Optionally extracts keywords if requested.
    """
    if history_messages is None:
        history_messages = []
    # Check and set response format for keyword extraction if needed
    keyword_extraction_flag = kwargs.pop("keyword_extraction", None)
    if keyword_extraction_flag:
        kwargs["response_format"] = "json"

    if "response_format" in kwargs:
        if isinstance(kwargs["response_format"], dict):
            kwargs["response_format"] = enforce_no_additional_properties(kwargs["response_format"])
        elif isinstance(kwargs["response_format"], str):
            pass
        else:
            kwargs["response_format"] = enforce_no_additional_properties(kwargs["response_format"].model_json_schema())  # oder .schema() in v1
     # kwargs["hashing_kv"].global_config["llm_model_name"]

    if any(x in model_name for x in ["mistral", "mixtral"]):
        kwargs.pop("response_format", None)

    return await litellm_complete_if_cache(
        model_name,
        prompt,
        system_prompt=system_prompt,
        history_messages=history_messages,
        **kwargs,
    )

litellm_complete_if_cache(model, prompt, system_prompt=None, history_messages=None, base_url=None, api_key=None, **kwargs) async

Core function to query the LiteLLM model. It builds the message context, invokes the completion API, and returns either a complete result string or an async iterator for streaming responses.

Source code in toolboxv2/mods/isaa/extras/adapter.py

@retry(
    stop=stop_after_attempt(3),
    wait=wait_exponential(multiplier=1, min=4, max=10),
    retry=retry_if_exception_type((RateLimitError, Timeout, APIConnectionError)),
)
async def litellm_complete_if_cache(
    model,
    prompt,
    system_prompt=None,
    history_messages=None,
    base_url=None,
    api_key=None,
    **kwargs,
) -> str | AsyncIterator[str]:
    """
    Core function to query the LiteLLM model. It builds the message context,
    invokes the completion API, and returns either a complete result string or
    an async iterator for streaming responses.
    """
    # Set the API key if provided
    if api_key:
        os.environ["LITELLM_API_KEY"] = api_key

    # Remove internal keys not needed for the client call
    kwargs.pop("hashing_kv", None)
    kwargs.pop("keyword_extraction", None)

    fallbacks_ = kwargs.pop("fallbacks", [])
    # Build the messages list from system prompt, conversation history, and the new prompt
    messages = []
    if system_prompt:
        messages.append({"role": "system", "content": system_prompt})
    if history_messages is not None:
        messages.extend(history_messages)
    messages.append({"role": "user", "content": prompt})

    # Log query details for debugging purposes
    try:
        # Depending on the response format, choose the appropriate API call
        if "response_format" in kwargs:
            response = await acompletion(
                model=model, messages=messages,
                fallbacks=fallbacks_+os.getenv("FALLBACKS_MODELS", '').split(','),
                **kwargs
            )
        else:
            response = await acompletion(
                model=model, messages=messages,
                fallbacks=os.getenv("FALLBACKS_MODELS", '').split(','),
                **kwargs
            )
    except Exception as e:
        print(f"\n{model=}\n{prompt=}\n{system_prompt=}\n{history_messages=}\n{base_url=}\n{api_key=}\n{kwargs=}")
        get_logger().error(f"Failed to litellm memory work {e}")
        return ""

    # Check if the response is a streaming response (i.e. an async iterator)
    if hasattr(response, "__aiter__"):

        async def inner():
            async for chunk in response:
                # Assume LiteLLM response structure is similar to OpenAI's
                content = chunk.choices[0].delta.content
                if content is None:
                    continue
                yield content

        return inner()
    else:
        # Non-streaming: extract and return the full content string

        content = response.choices[0].message.content
        if content is None:
            content = response.choices[0].message.tool_calls[0].function.arguments
        return content

litellm_embed(texts, model='gemini/text-embedding-004', dimensions=256, base_url=None, api_key=None) async

Generates embeddings for the given list of texts using LiteLLM.

Source code in toolboxv2/mods/isaa/extras/adapter.py

@retry(
    stop=stop_after_attempt(3),
    wait=wait_exponential(multiplier=1, min=4, max=60),
    retry=retry_if_exception_type((RateLimitError, Timeout, APIConnectionError)),
)
async def litellm_embed(
    texts: list[str],
    model: str = "gemini/text-embedding-004",
    dimensions: int = 256,
    base_url: str = None,
    api_key: str = None,
) -> np.ndarray:
    """
    Generates embeddings for the given list of texts using LiteLLM.
    """
    response = await litellm.aembedding(
        model=model, input=texts,
        dimensions=dimensions,
        # encoding_format="float"
    )
    return np.array([dp.embedding for dp in response.data])

cahin_printer

ChainPrinter

Custom printer for enhanced chain visualization and progress display

Source code in toolboxv2/mods/isaa/extras/cahin_printer.py

class ChainPrinter:
    """Custom printer for enhanced chain visualization and progress display"""

    def __init__(self, verbose: bool = True):
        self.verbose = verbose
        self.colors = {
            'success': '\033[92m',
            'error': '\033[91m',
            'warning': '\033[93m',
            'info': '\033[94m',
            'highlight': '\033[95m',
            'dim': '\033[2m',
            'bold': '\033[1m',
            'reset': '\033[0m'
        }

    def _colorize(self, text: str, color: str) -> str:
        return f"{self.colors.get(color, '')}{text}{self.colors['reset']}"

    def print_header(self, title: str, subtitle: str = None):
        """Print formatted header"""
        print(f"\n{self._colorize('═' * 60, 'highlight')}")
        print(f"{self._colorize(f'🔗 {title}', 'bold')}")
        if subtitle:
            print(f"{self._colorize(subtitle, 'dim')}")
        print(f"{self._colorize('═' * 60, 'highlight')}\n")

    def print_success(self, message: str):
        print(f"{self._colorize('✅ ', 'success')}{message}")

    def print_error(self, message: str):
        print(f"{self._colorize('❌ ', 'error')}{message}")

    def print_warning(self, message: str):
        print(f"{self._colorize('⚠️ ', 'warning')}{message}")

    def print_info(self, message: str):
        print(f"{self._colorize('ℹ️ ', 'info')}{message}")

    def print_progress_start(self, chain_name: str):
        print(f"\n{self._colorize('🚀 Starting chain execution:', 'info')} {self._colorize(chain_name, 'bold')}")

    def print_task_start(self, task_name: str, current: int, total: int):
        progress = f"[{current + 1}/{total}]" if total > 0 else ""
        print(f"  {self._colorize('▶️ ', 'info')}{progress} {task_name}")

    def print_task_complete(self, task_name: str, completed: int, total: int):
        progress = f"[{completed}/{total}]" if total > 0 else ""
        print(f"  {self._colorize('✅', 'success')} {progress} {task_name} completed")

    def print_task_error(self, task_name: str, error: str):
        print(f"  {self._colorize('❌', 'error')} {task_name} failed: {error}")

    def print_progress_end(self, chain_name: str, duration: float, success: bool):
        status = self._colorize('✅ COMPLETED', 'success') if success else self._colorize('❌ FAILED', 'error')
        print(f"\n{status} {chain_name} ({duration:.2f}s)\n")

    def print_tool_usage_success(self, tool_name: str, duration: float, is_meta_tool: bool = False, tool_args: dict[str, Any] = None):
        if is_meta_tool:
            print(f"  {self._colorize('🔧 ', 'info')}{tool_name} completed ({duration:.2f}s) {arguments_summary(tool_args)}")
        else:
            print(f"  {self._colorize('🔩 ', 'info')}{tool_name} completed ({duration:.2f}s) {arguments_summary(tool_args)}")

    def print_tool_usage_error(self, tool_name: str, error: str, is_meta_tool: bool = False):
        if is_meta_tool:
            print(f"  {self._colorize('🔧 ', 'error')}{tool_name} failed: {error}")
        else:
            print(f"  {self._colorize('🔩 ', 'error')}{tool_name} failed: {error}")

    def print_outline_created(self, outline: dict):
        for step in outline.get("steps", []):
            print(f"  {self._colorize('📖 ', 'info')}Step: {self._colorize(step.get('description', 'Unknown'), 'dim')}")

    def print_reasoning_loop(self, loop_data: dict):
        print(f"  {self._colorize('🧠 ', 'info')}Reasoning Loop #{loop_data.get('loop_number', '?')}")
        print(
            f"    {self._colorize('📖 ', 'info')}Outline Step: {loop_data.get('outline_step', 0)} of {loop_data.get('outline_total', 0)}")
        print(f"    {self._colorize('📚 ', 'info')}Context Size: {loop_data.get('context_size', 0)} entries")
        print(f"    {self._colorize('📋 ', 'info')}Task Stack: {loop_data.get('task_stack_size', 0)} items")
        print(f"    {self._colorize('🔄 ', 'info')}Recovery Attempts: {loop_data.get('auto_recovery_attempts', 0)}")
        print(f"    {self._colorize('📊 ', 'info')}Performance Metrics: {loop_data.get('performance_metrics', {})}")

    def print_chain_list(self, chains: list[tuple[str, ChainMetadata]]):
        """Print formatted list of available chains"""
        if not chains:
            self.print_info("No chains found. Use 'create' to build your first chain.")
            return

        self.print_header("Available Chains", f"Total: {len(chains)}")

        for name, meta in chains:
            # Status indicators
            indicators = []
            if meta.has_parallels:
                indicators.append(self._colorize("⚡", "highlight"))
            if meta.has_conditionals:
                indicators.append(self._colorize("🔀", "warning"))
            if meta.has_error_handling:
                indicators.append(self._colorize("🛡️", "info"))

            status_str = " ".join(indicators) if indicators else ""

            # Complexity color
            complexity_colors = {"simple": "success", "medium": "warning", "complex": "error"}
            complexity = self._colorize(meta.complexity, complexity_colors.get(meta.complexity, "info"))

            print(f"  {self._colorize(name, 'bold')} {status_str}")
            print(f"    {meta.description or 'No description'}")
            print(f"    {complexity} • {meta.agent_count} agents • {meta.version}")
            if meta.tags:
                tags_str = " ".join([f"#{tag}" for tag in meta.tags])
                print(f"    {self._colorize(tags_str, 'dim')}")
            print()

print_chain_list(chains)

Print formatted list of available chains

Source code in toolboxv2/mods/isaa/extras/cahin_printer.py

def print_chain_list(self, chains: list[tuple[str, ChainMetadata]]):
    """Print formatted list of available chains"""
    if not chains:
        self.print_info("No chains found. Use 'create' to build your first chain.")
        return

    self.print_header("Available Chains", f"Total: {len(chains)}")

    for name, meta in chains:
        # Status indicators
        indicators = []
        if meta.has_parallels:
            indicators.append(self._colorize("⚡", "highlight"))
        if meta.has_conditionals:
            indicators.append(self._colorize("🔀", "warning"))
        if meta.has_error_handling:
            indicators.append(self._colorize("🛡️", "info"))

        status_str = " ".join(indicators) if indicators else ""

        # Complexity color
        complexity_colors = {"simple": "success", "medium": "warning", "complex": "error"}
        complexity = self._colorize(meta.complexity, complexity_colors.get(meta.complexity, "info"))

        print(f"  {self._colorize(name, 'bold')} {status_str}")
        print(f"    {meta.description or 'No description'}")
        print(f"    {complexity} • {meta.agent_count} agents • {meta.version}")
        if meta.tags:
            tags_str = " ".join([f"#{tag}" for tag in meta.tags])
            print(f"    {self._colorize(tags_str, 'dim')}")
        print()

print_header(title, subtitle=None)

Print formatted header

Source code in toolboxv2/mods/isaa/extras/cahin_printer.py

def print_header(self, title: str, subtitle: str = None):
    """Print formatted header"""
    print(f"\n{self._colorize('═' * 60, 'highlight')}")
    print(f"{self._colorize(f'🔗 {title}', 'bold')}")
    if subtitle:
        print(f"{self._colorize(subtitle, 'dim')}")
    print(f"{self._colorize('═' * 60, 'highlight')}\n")

ChainProgressTracker

Enhanced progress tracker for chain execution with live display

Source code in toolboxv2/mods/isaa/extras/cahin_printer.py

class ChainProgressTracker:
    """Enhanced progress tracker for chain execution with live display"""

    def __init__(self, chain_printer: 'ChainPrinter' = None):
        self.events: list[ProgressEvent] = []
        self.start_time = time.time()
        self.chain_printer = chain_printer or ChainPrinter()
        self.current_task = None
        self.task_count = 0
        self.completed_tasks = 0

    async def emit_event(self, event: ProgressEvent):
        """Emit progress event with live display updates"""
        self.events.append(event)

        if event.event_type == "chain_start":
            self.task_count = event.metadata.get("task_count", 0)
            self.chain_printer.print_progress_start(event.node_name)

        elif event.event_type == "task_start":
            self.current_task = event.node_name
            self.chain_printer.print_task_start(event.node_name, self.completed_tasks, self.task_count)

        elif event.event_type == "task_complete":
            if event.status == NodeStatus.COMPLETED:
                self.completed_tasks += 1
                self.chain_printer.print_task_complete(event.node_name, self.completed_tasks, self.task_count)
            elif event.status == NodeStatus.FAILED:
                self.chain_printer.print_task_error(event.node_name, event.metadata.get("error", "Unknown error"))

        elif event.event_type == "chain_end":
            duration = time.time() - self.start_time
            self.chain_printer.print_progress_end(event.node_name, duration, event.status == NodeStatus.COMPLETED)

        elif event.event_type == "tool_call" and event.success == False:
            self.chain_printer.print_tool_usage_error(event.tool_name, event.metadata.get("error",
                                                                                          event.metadata.get("message",
                                                                                                             event.error_details.get(
                                                                                                                 "error",
                                                                                                                 "Unknown error"))))

        elif event.event_type == "tool_call" and event.success == True:
            self.chain_printer.print_tool_usage_success(event.tool_name, event.duration, event.is_meta_tool, event.tool_args)

        elif event.event_type == "outline_created":
            self.chain_printer.print_outline_created(event.metadata.get("outline", {}))

        elif event.event_type == "reasoning_loop":
            self.chain_printer.print_reasoning_loop(event.metadata)

        elif event.event_type == "task_error":
            self.chain_printer.print_task_error(event.node_name, event.metadata.get("error", "Unknown error"))

emit_event(event) async

Emit progress event with live display updates

Source code in toolboxv2/mods/isaa/extras/cahin_printer.py

async def emit_event(self, event: ProgressEvent):
    """Emit progress event with live display updates"""
    self.events.append(event)

    if event.event_type == "chain_start":
        self.task_count = event.metadata.get("task_count", 0)
        self.chain_printer.print_progress_start(event.node_name)

    elif event.event_type == "task_start":
        self.current_task = event.node_name
        self.chain_printer.print_task_start(event.node_name, self.completed_tasks, self.task_count)

    elif event.event_type == "task_complete":
        if event.status == NodeStatus.COMPLETED:
            self.completed_tasks += 1
            self.chain_printer.print_task_complete(event.node_name, self.completed_tasks, self.task_count)
        elif event.status == NodeStatus.FAILED:
            self.chain_printer.print_task_error(event.node_name, event.metadata.get("error", "Unknown error"))

    elif event.event_type == "chain_end":
        duration = time.time() - self.start_time
        self.chain_printer.print_progress_end(event.node_name, duration, event.status == NodeStatus.COMPLETED)

    elif event.event_type == "tool_call" and event.success == False:
        self.chain_printer.print_tool_usage_error(event.tool_name, event.metadata.get("error",
                                                                                      event.metadata.get("message",
                                                                                                         event.error_details.get(
                                                                                                             "error",
                                                                                                             "Unknown error"))))

    elif event.event_type == "tool_call" and event.success == True:
        self.chain_printer.print_tool_usage_success(event.tool_name, event.duration, event.is_meta_tool, event.tool_args)

    elif event.event_type == "outline_created":
        self.chain_printer.print_outline_created(event.metadata.get("outline", {}))

    elif event.event_type == "reasoning_loop":
        self.chain_printer.print_reasoning_loop(event.metadata)

    elif event.event_type == "task_error":
        self.chain_printer.print_task_error(event.node_name, event.metadata.get("error", "Unknown error"))

mcp_session_manager

MCPSessionManager

Manages persistent MCP sessions with automatic reconnection and parallel processing

Source code in toolboxv2/mods/isaa/extras/mcp_session_manager.py

class MCPSessionManager:
    """Manages persistent MCP sessions with automatic reconnection and parallel processing"""

    def __init__(self):
        self.sessions: dict[str, ClientSession] = {}
        self.connections: dict[str, Any] = {}
        self.capabilities_cache: dict[str, dict] = {}
        self.retry_count: dict[str, int] = {}
        self.max_retries = 3
        self.connection_timeout = 15.0  # 10 seconds timeout
        self.operation_timeout = 10.0  # 5 seconds for operations

    async def get_session_with_timeout(self, server_name: str, server_config: dict[str, Any]) -> ClientSession | None:
        """Get session with timeout protection"""
        try:
            return await asyncio.wait_for(
                self.get_session(server_name, server_config),
                timeout=self.connection_timeout
            )
        except TimeoutError:
            eprint(f"MCP session creation timeout for {server_name}")
            return None

    async def get_session(self, server_name: str, server_config: dict[str, Any]) -> ClientSession | None:
        """Get or create persistent MCP session with proper context management"""
        if server_name in self.sessions:
            try:
                # Test if session is still alive with timeout
                session = self.sessions[server_name]
                # Quick connectivity test
                await asyncio.wait_for(session.list_tools(), timeout=2.0)
                return session
            except Exception as e:
                wprint(f"MCP session {server_name} failed, recreating: {e}")
                # Clean up the old session
                if server_name in self.sessions:
                    del self.sessions[server_name]
                if server_name in self.connections:
                    del self.connections[server_name]

        return await self._create_session(server_name, server_config)

    async def _create_session(self, server_name: str, server_config: dict[str, Any]) -> ClientSession | None:
        """Create new MCP session with improved error handling"""
        try:
            command = server_config.get('command')
            args = server_config.get('args', [])
            env = server_config.get('env', {})
            transport_type = server_config.get('transport', 'stdio')

            if not command:
                eprint(f"No command specified for MCP server {server_name}")
                return None

            iprint(f"Creating MCP session for {server_name} (transport: {transport_type})")

            session = None

            # Create connection based on transport type
            if transport_type == 'stdio':
                session = await self._create_stdio_session(server_name, command, args, env)
            elif transport_type in ['http', 'streamable-http']:
                session = await self._create_http_session(server_name, server_config)
            else:
                eprint(f"Unsupported transport type: {transport_type}")
                return None

            if session:
                self.sessions[server_name] = session
                self.retry_count[server_name] = 0
                iprint(f"✓ MCP session created successfully: {server_name}")
                return session

            return None

        except Exception as e:
            self.retry_count[server_name] = self.retry_count.get(server_name, 0) + 1
            if self.retry_count[server_name] <= self.max_retries:
                wprint(f"MCP session creation failed (attempt {self.retry_count[server_name]}/{self.max_retries}): {e}")
                await asyncio.sleep(1.0)  # Longer delay before retry
                return await self._create_session(server_name, server_config)
            else:
                eprint(f"✗ MCP session creation failed after {self.max_retries} attempts: {e}")
                return None

    async def _create_stdio_session(self, server_name: str, command: str, args: list[str], env: dict[str, str]) -> \
    ClientSession | None:
        """Create stdio MCP session with fixed async context handling"""
        try:
            from mcp import StdioServerParameters
            from mcp.client.stdio import stdio_client

            # Prepare environment
            process_env = os.environ.copy()
            process_env.update(env)

            server_params = StdioServerParameters(
                command=command,
                args=args,
                env=process_env
            )

            # Create the stdio client and session in a single task context
            stdio_connection = stdio_client(server_params)

            # Enter the context manager
            read_stream, write_stream = await stdio_connection.__aenter__()

            # Store the connection for cleanup later
            self.connections[server_name] = stdio_connection

            # Create session
            session = ClientSession(read_stream, write_stream)

            # Initialize session in the same context
            await session.__aenter__()
            await asyncio.wait_for(session.initialize(), timeout=self.connection_timeout)

            return session

        except Exception as e:
            eprint(f"Failed to create stdio session for {server_name}: {e}")
            # Cleanup on failure
            if server_name in self.connections:
                with contextlib.suppress(Exception):
                    await self.connections[server_name].__aexit__(None, None, None)
                del self.connections[server_name]
            return None

    async def _create_http_session(self, server_name: str, server_config: dict[str, Any]) -> ClientSession | None:
        """Create HTTP MCP session with timeout"""
        try:
            from mcp.client.streamable_http import streamablehttp_client

            url = server_config.get('url', f"http://localhost:{server_config.get('port', 8000)}/mcp")

            connection = streamablehttp_client(url)
            read_stream, write_stream, cleanup = await asyncio.wait_for(
                connection.__aenter__(),
                timeout=self.connection_timeout
            )

            session = ClientSession(read_stream, write_stream)
            await session.__aenter__()
            await asyncio.wait_for(
                session.initialize(),
                timeout=self.connection_timeout
            )

            self.connections[server_name] = connection
            return session

        except Exception as e:
            eprint(f"Failed to create HTTP session for {server_name}: {e}")
            return None

    async def extract_capabilities_with_timeout(self, session: ClientSession, server_name: str) -> dict[str, dict]:
        """Extract capabilities with timeout protection"""
        try:
            return await asyncio.wait_for(
                self.extract_capabilities(session, server_name),
                timeout=self.operation_timeout
            )
        except TimeoutError:
            eprint(f"Capability extraction timeout for {server_name}")
            return {'tools': {}, 'resources': {}, 'resource_templates': {}, 'prompts': {}, 'images': {}}

    async def extract_capabilities(self, session: ClientSession, server_name: str) -> dict[str, dict]:
        """Extract all capabilities from MCP session"""
        if server_name in self.capabilities_cache:
            return self.capabilities_cache[server_name]

        capabilities = {
            'tools': {},
            'resources': {},
            'resource_templates': {},
            'prompts': {},
            'images': {}
        }

        try:
            # Extract tools with individual timeouts
            try:
                tools_response = await asyncio.wait_for(session.list_tools(), timeout=3.0)
                for tool in tools_response.tools:
                    capabilities['tools'][tool.name] = {
                        'name': tool.name,
                        'description': tool.description or '',
                        'input_schema': tool.inputSchema,
                        'output_schema': getattr(tool, 'outputSchema', None),
                        'display_name': getattr(tool, 'title', tool.name)
                    }
            except TimeoutError:
                wprint(f"Tools extraction timeout for {server_name}")
            except Exception as e:
                wprint(f"Failed to extract tools from {server_name}: {e}")

            # Extract resources with timeout
            try:
                resources_response = await asyncio.wait_for(session.list_resources(), timeout=3.0)
                for resource in resources_response.resources:
                    capabilities['resources'][str(resource.uri)] = {
                        'uri': str(resource.uri),
                        'name': resource.name or str(resource.uri),
                        'description': resource.description or '',
                        'mime_type': getattr(resource, 'mimeType', None)
                    }
            except TimeoutError:
                wprint(f"Resources extraction timeout for {server_name}")
            except Exception as e:
                wprint(f"Failed to extract resources from {server_name}: {e}")

            # Extract resource templates with timeout
            try:
                templates_response = await asyncio.wait_for(session.list_resource_templates(), timeout=3.0)
                for template in templates_response.resourceTemplates:
                    capabilities['resource_templates'][template.uriTemplate] = {
                        'uri_template': template.uriTemplate,
                        'name': template.name or template.uriTemplate,
                        'description': template.description or ''
                    }
            except TimeoutError:
                wprint(f"Resource templates extraction timeout for {server_name}")
            except Exception as e:
                wprint(f"Failed to extract resource templates from {server_name}: {e}")

            # Extract prompts with timeout
            try:
                prompts_response = await asyncio.wait_for(session.list_prompts(), timeout=3.0)
                for prompt in prompts_response.prompts:
                    capabilities['prompts'][prompt.name] = {
                        'name': prompt.name,
                        'description': prompt.description or '',
                        'arguments': [
                            {
                                'name': arg.name,
                                'description': arg.description or '',
                                'required': arg.required
                            } for arg in (prompt.arguments or [])
                        ]
                    }
            except TimeoutError:
                wprint(f"Prompts extraction timeout for {server_name}")
            except Exception as e:
                wprint(f"Failed to extract prompts from {server_name}: {e}")

            self.capabilities_cache[server_name] = capabilities

            total_caps = (len(capabilities['tools']) + len(capabilities['resources']) +
                          len(capabilities['resource_templates']) + len(capabilities['prompts']))
            iprint(f"✓ Extracted {total_caps} capabilities from {server_name}")

        except Exception as e:
            eprint(f"Failed to extract capabilities from {server_name}: {e}")

        return capabilities

    async def _cleanup_session(self, server_name: str):
        """Clean up a specific session with proper context management"""
        try:
            # Clean up session first
            if server_name in self.sessions:
                try:
                    session = self.sessions[server_name]
                    await asyncio.wait_for(session.__aexit__(None, None, None), timeout=2.0)
                except (TimeoutError, Exception) as e:
                    wprint(f"Session cleanup warning for {server_name}: {e}")
                finally:
                    del self.sessions[server_name]

            # Clean up connection
            if server_name in self.connections:
                try:
                    connection = self.connections[server_name]
                    await asyncio.wait_for(connection.__aexit__(None, None, None), timeout=2.0)
                except (TimeoutError, Exception) as e:
                    wprint(f"Connection cleanup warning for {server_name}: {e}")
                finally:
                    del self.connections[server_name]

            # Clear cache
            if server_name in self.capabilities_cache:
                del self.capabilities_cache[server_name]

            # Reset retry count
            if server_name in self.retry_count:
                del self.retry_count[server_name]

        except Exception as e:
            wprint(f"Cleanup error for {server_name}: {e}")

    async def cleanup_all(self):
        """Clean up all sessions with timeout and proper error handling"""
        cleanup_tasks = []
        for server_name in list(self.sessions.keys()):
            task = asyncio.create_task(self._cleanup_session(server_name))
            cleanup_tasks.append(task)

        if cleanup_tasks:
            try:
                # Use gather with return_exceptions=True to collect all results
                results = await asyncio.wait_for(
                    asyncio.gather(*cleanup_tasks, return_exceptions=True),
                    timeout=5.0
                )

                # Log any non-cancellation exceptions
                for i, result in enumerate(results):
                    if isinstance(result, Exception) and not isinstance(result, asyncio.CancelledError):
                        wprint(f"Error cleaning up MCP session: {result}")

            except (asyncio.TimeoutError, asyncio.CancelledError):
                # Handle timeout and cancellation
                wprint("MCP session cleanup timeout or cancelled")
                # Cancel all tasks
                for task in cleanup_tasks:
                    if not task.done():
                        task.cancel()

                # Give tasks a moment to cancel cleanly
                try:
                    await asyncio.wait(cleanup_tasks, timeout=1.0)
                except asyncio.CancelledError:
                    pass

            except Exception as e:
                wprint(f"Unexpected error during MCP session cleanup: {e}")
                # Cancel remaining tasks
                for task in cleanup_tasks:
                    if not task.done():
                        task.cancel()

cleanup_all() async

Clean up all sessions with timeout and proper error handling

Source code in toolboxv2/mods/isaa/extras/mcp_session_manager.py

async def cleanup_all(self):
    """Clean up all sessions with timeout and proper error handling"""
    cleanup_tasks = []
    for server_name in list(self.sessions.keys()):
        task = asyncio.create_task(self._cleanup_session(server_name))
        cleanup_tasks.append(task)

    if cleanup_tasks:
        try:
            # Use gather with return_exceptions=True to collect all results
            results = await asyncio.wait_for(
                asyncio.gather(*cleanup_tasks, return_exceptions=True),
                timeout=5.0
            )

            # Log any non-cancellation exceptions
            for i, result in enumerate(results):
                if isinstance(result, Exception) and not isinstance(result, asyncio.CancelledError):
                    wprint(f"Error cleaning up MCP session: {result}")

        except (asyncio.TimeoutError, asyncio.CancelledError):
            # Handle timeout and cancellation
            wprint("MCP session cleanup timeout or cancelled")
            # Cancel all tasks
            for task in cleanup_tasks:
                if not task.done():
                    task.cancel()

            # Give tasks a moment to cancel cleanly
            try:
                await asyncio.wait(cleanup_tasks, timeout=1.0)
            except asyncio.CancelledError:
                pass

        except Exception as e:
            wprint(f"Unexpected error during MCP session cleanup: {e}")
            # Cancel remaining tasks
            for task in cleanup_tasks:
                if not task.done():
                    task.cancel()

extract_capabilities(session, server_name) async

Extract all capabilities from MCP session

Source code in toolboxv2/mods/isaa/extras/mcp_session_manager.py

async def extract_capabilities(self, session: ClientSession, server_name: str) -> dict[str, dict]:
    """Extract all capabilities from MCP session"""
    if server_name in self.capabilities_cache:
        return self.capabilities_cache[server_name]

    capabilities = {
        'tools': {},
        'resources': {},
        'resource_templates': {},
        'prompts': {},
        'images': {}
    }

    try:
        # Extract tools with individual timeouts
        try:
            tools_response = await asyncio.wait_for(session.list_tools(), timeout=3.0)
            for tool in tools_response.tools:
                capabilities['tools'][tool.name] = {
                    'name': tool.name,
                    'description': tool.description or '',
                    'input_schema': tool.inputSchema,
                    'output_schema': getattr(tool, 'outputSchema', None),
                    'display_name': getattr(tool, 'title', tool.name)
                }
        except TimeoutError:
            wprint(f"Tools extraction timeout for {server_name}")
        except Exception as e:
            wprint(f"Failed to extract tools from {server_name}: {e}")

        # Extract resources with timeout
        try:
            resources_response = await asyncio.wait_for(session.list_resources(), timeout=3.0)
            for resource in resources_response.resources:
                capabilities['resources'][str(resource.uri)] = {
                    'uri': str(resource.uri),
                    'name': resource.name or str(resource.uri),
                    'description': resource.description or '',
                    'mime_type': getattr(resource, 'mimeType', None)
                }
        except TimeoutError:
            wprint(f"Resources extraction timeout for {server_name}")
        except Exception as e:
            wprint(f"Failed to extract resources from {server_name}: {e}")

        # Extract resource templates with timeout
        try:
            templates_response = await asyncio.wait_for(session.list_resource_templates(), timeout=3.0)
            for template in templates_response.resourceTemplates:
                capabilities['resource_templates'][template.uriTemplate] = {
                    'uri_template': template.uriTemplate,
                    'name': template.name or template.uriTemplate,
                    'description': template.description or ''
                }
        except TimeoutError:
            wprint(f"Resource templates extraction timeout for {server_name}")
        except Exception as e:
            wprint(f"Failed to extract resource templates from {server_name}: {e}")

        # Extract prompts with timeout
        try:
            prompts_response = await asyncio.wait_for(session.list_prompts(), timeout=3.0)
            for prompt in prompts_response.prompts:
                capabilities['prompts'][prompt.name] = {
                    'name': prompt.name,
                    'description': prompt.description or '',
                    'arguments': [
                        {
                            'name': arg.name,
                            'description': arg.description or '',
                            'required': arg.required
                        } for arg in (prompt.arguments or [])
                    ]
                }
        except TimeoutError:
            wprint(f"Prompts extraction timeout for {server_name}")
        except Exception as e:
            wprint(f"Failed to extract prompts from {server_name}: {e}")

        self.capabilities_cache[server_name] = capabilities

        total_caps = (len(capabilities['tools']) + len(capabilities['resources']) +
                      len(capabilities['resource_templates']) + len(capabilities['prompts']))
        iprint(f"✓ Extracted {total_caps} capabilities from {server_name}")

    except Exception as e:
        eprint(f"Failed to extract capabilities from {server_name}: {e}")

    return capabilities

extract_capabilities_with_timeout(session, server_name) async

Extract capabilities with timeout protection

Source code in toolboxv2/mods/isaa/extras/mcp_session_manager.py

async def extract_capabilities_with_timeout(self, session: ClientSession, server_name: str) -> dict[str, dict]:
    """Extract capabilities with timeout protection"""
    try:
        return await asyncio.wait_for(
            self.extract_capabilities(session, server_name),
            timeout=self.operation_timeout
        )
    except TimeoutError:
        eprint(f"Capability extraction timeout for {server_name}")
        return {'tools': {}, 'resources': {}, 'resource_templates': {}, 'prompts': {}, 'images': {}}

get_session(server_name, server_config) async

Get or create persistent MCP session with proper context management

Source code in toolboxv2/mods/isaa/extras/mcp_session_manager.py

async def get_session(self, server_name: str, server_config: dict[str, Any]) -> ClientSession | None:
    """Get or create persistent MCP session with proper context management"""
    if server_name in self.sessions:
        try:
            # Test if session is still alive with timeout
            session = self.sessions[server_name]
            # Quick connectivity test
            await asyncio.wait_for(session.list_tools(), timeout=2.0)
            return session
        except Exception as e:
            wprint(f"MCP session {server_name} failed, recreating: {e}")
            # Clean up the old session
            if server_name in self.sessions:
                del self.sessions[server_name]
            if server_name in self.connections:
                del self.connections[server_name]

    return await self._create_session(server_name, server_config)

get_session_with_timeout(server_name, server_config) async

Get session with timeout protection

Source code in toolboxv2/mods/isaa/extras/mcp_session_manager.py

async def get_session_with_timeout(self, server_name: str, server_config: dict[str, Any]) -> ClientSession | None:
    """Get session with timeout protection"""
    try:
        return await asyncio.wait_for(
            self.get_session(server_name, server_config),
            timeout=self.connection_timeout
        )
    except TimeoutError:
        eprint(f"MCP session creation timeout for {server_name}")
        return None

modes

generate_prompt(subject, context='', additional_requirements=None)

Generates a prompt based on the given subject, with optional context and additional requirements.

Parameters: - subject (str): The main subject for the prompt. - context (str): Optional additional context to tailor the prompt. - additional_requirements (Dict[str, Any]): Optional additional parameters or requirements for the prompt.

Returns: - str: A crafted prompt.

Source code in toolboxv2/mods/isaa/extras/modes.py

def generate_prompt(subject: str, context: str = "", additional_requirements: dict[str, Any] = None) -> str:
    """
    Generates a prompt based on the given subject, with optional context and additional requirements.

    Parameters:
    - subject (str): The main subject for the prompt.
    - context (str): Optional additional context to tailor the prompt.
    - additional_requirements (Dict[str, Any]): Optional additional parameters or requirements for the prompt.

    Returns:
    - str: A crafted prompt.
    """
    prompt = f"Based on the subject '{subject}', with the context '{context}', generate a clear and precise instruction."
    if additional_requirements:
        prompt += f" Consider the following requirements: {additional_requirements}."
    return prompt

terminal_progress

AgentExecutionState

Verwaltet den gesamten Zustand des Agentenablaufs, um eine reichhaltige Visualisierung zu ermöglichen.

Source code in toolboxv2/mods/isaa/extras/terminal_progress.py

class AgentExecutionState:
    """
    Verwaltet den gesamten Zustand des Agentenablaufs, um eine reichhaltige
    Visualisierung zu ermöglichen.
    """

    def __init__(self):
        self.agent_name = "Agent"
        self.execution_phase = 'initializing'
        self.start_time = time.time()
        self.error_count = 0
        self.outline = None
        self.outline_progress = {'current_step': 0, 'total_steps': 0}
        self.reasoning_notes = []
        self.current_reasoning_loop = 0
        self.active_delegation = None
        self.active_task_plan = None
        self.tool_history = []
        self.llm_interactions = {'total_calls': 0, 'total_cost': 0.0, 'total_tokens': 0}
        self.active_nodes = set()
        self.node_flow = []
        self.last_event_per_node = {}
        self.event_count = 0

ProgressiveTreePrinter

Eine moderne, produktionsreife Terminal-Visualisierung für den Agenten-Ablauf.

Source code in toolboxv2/mods/isaa/extras/terminal_progress.py

class ProgressiveTreePrinter:
    """Eine moderne, produktionsreife Terminal-Visualisierung für den Agenten-Ablauf."""

    def __init__(self, **kwargs):
        self.processor = StateProcessor()
        self.style = Style()
        self.llm_stream_chunks = ""
        self.buffer = 0
        self._display_interval = 0.1
        self._last_update_time = time.time()
        self._terminal_width = 80
        self._terminal_height = 24
        self._is_initialized = False

        # Terminal-Größe ermitteln
        self._update_terminal_size()

        # Original print sichern
        import builtins
        self._original_print = builtins.print
        builtins.print = self.print
        self._terminal_content = []  # List für O(1) append


    def print(self, *args, **kwargs):
        """
        Überladene print Funktion die automatisch Content speichert
        """
        # Capture output in StringIO für Effizienz
        output = StringIO()
        if 'file' in kwargs:
            del kwargs['file']
        self._original_print(*args, file=output, **kwargs)
        content = output.getvalue()

        # Speichere nur wenn content nicht leer
        if content.strip():
            self._terminal_content.append(content.rstrip('\n'))

        # Normale Ausgabe
        self._original_print(*args, **kwargs)

    def live_print(self,*args, **kwargs):
        """
        Live print ohne Content-Speicherung für temporäre Ausgaben
        """
        self._original_print(*args, **kwargs)

    @staticmethod
    def clear():
        """
        Speichert aktuellen Terminal-Content und cleared das Terminal
        Systemagnostisch (Windows/Unix)
        """
        # Clear terminal - systemagnostisch
        if os.name == 'nt':  # Windows
            os.system('cls')
        else:  # Unix/Linux/macOS
            os.system('clear')

    def restore_content(self):
        """
        Stellt den gespeicherten Terminal-Content in einer Aktion wieder her
        Effizient durch join operation
        """
        if self._terminal_content:
            # Effiziente Wiederherstellung mit join
            restored_output = '\n'.join(self._terminal_content)
            self._original_print(restored_output)

    def _update_terminal_size(self):
        """Aktualisiert die Terminal-Dimensionen."""
        try:
            terminal_size = shutil.get_terminal_size()
            self._terminal_width = max(terminal_size.columns, 80)
            self._terminal_height = max(terminal_size.lines, 24)
        except:
            self._terminal_width = 80
            self._terminal_height = 24

    def _truncate_text(self, text: str, max_length: int) -> str:
        """Kürzt Text auf maximale Länge und fügt '...' hinzu."""
        if len(remove_styles(text)) <= max_length:
            return text

        # Berücksichtige Style-Codes beim Kürzen
        plain_text = remove_styles(text)
        if len(plain_text) > max_length - 3:
            truncated = plain_text[:max_length - 3] + "..."
            return truncated
        return text

    def _fit_content_to_terminal(self, lines: list) -> list:
        """Passt den Inhalt an die Terminal-Größe an."""
        fitted_lines = []
        available_width = self._terminal_width - 2  # Rand lassen

        for line in lines:
            if len(remove_styles(line)) > available_width:
                fitted_lines.append(self._truncate_text(line, available_width))
            else:
                fitted_lines.append(line)

        # Wenn zu viele Zeilen, die wichtigsten behalten
        max_lines = self._terminal_height - 3  # Platz für Header und Eingabezeile
        if len(fitted_lines) > max_lines:
            # Header behalten, dann die letzten Zeilen
            header_lines = fitted_lines[:5]  # Erste 5 Zeilen (Header)
            remaining_lines = fitted_lines[5:]

            if len(header_lines) < max_lines:
                content_space = max_lines - len(header_lines)
                fitted_lines = header_lines + remaining_lines[-content_space:]
            else:
                fitted_lines = fitted_lines[:max_lines]

        return fitted_lines

    async def progress_callback(self, event: ProgressEvent):
        """Haupteingangspunkt für Progress Events."""
        if event.event_type == 'execution_start':
            self.processor = StateProcessor()
            self._is_initialized = True


        self.processor.process_event(event)

        # LLM Stream Handling
        if event.event_type == 'llm_stream_chunk':
            self.llm_stream_chunks += event.llm_output
            # Stream-Chunks auf vernünftige Größe begrenzen
            lines = self.llm_stream_chunks.replace('\\n', '\n').split('\n')
            if len(lines) > 8:
                self.llm_stream_chunks = '\n'.join(lines[-8:])
            self.buffer += 1
            if self.buffer > 5:
                self.buffer = 0
            else:
                return

        if event.event_type == 'llm_call':
            self.llm_stream_chunks = ""

        # Display nur bei wichtigen Events oder zeitbasiert aktualisieren
        should_update = (
            time.time() - self._last_update_time > self._display_interval or
            event.event_type in ['execution_complete', 'outline_created', 'plan_created', 'node_enter']
        )

        if should_update and self._is_initialized:
            self._update_display()
            self._last_update_time = time.time()


        if event.event_type in ['execution_complete', 'error']:
            self.restore_content()
            self.print_final_summary()

    def _update_display(self):
        """Aktualisiert die Anzeige im Terminal."""
        self._update_terminal_size()  # Terminal-Größe neu ermitteln
        output_lines = self._render_full_display()

        self.clear()
        self.live_print('\n'.join(output_lines))


    def _render_full_display(self) -> list:
        """Rendert die komplette Anzeige als Liste von Zeilen."""
        state = self.processor.state
        all_lines = []

        # Header
        header_lines = self._render_header(state).split('\n')
        all_lines.extend(header_lines)
        all_lines.append("")  # Leerzeile

        # Hauptinhalt basierend auf Ausführungsphase
        if state.outline:
            outline_content = self._render_outline_section(state)
            if outline_content:
                all_lines.extend(outline_content)
                all_lines.append("")

        reasoning_content = self._render_reasoning_section(state)
        if reasoning_content:
            all_lines.extend(reasoning_content)
            all_lines.append("")

        activity_content = self._render_activity_section(state)
        if activity_content:
            all_lines.extend(activity_content)
            all_lines.append("")

        if state.active_task_plan:
            plan_content = self._render_task_plan_section(state)
            if plan_content:
                all_lines.extend(plan_content)
                all_lines.append("")

        if state.tool_history:
            tool_content = self._render_tool_history_section(state)
            if tool_content:
                all_lines.extend(tool_content)
                all_lines.append("")

        system_content = self._render_system_flow_section(state)
        if system_content:
            all_lines.extend(system_content)

        # An Terminal-Größe anpassen
        return self._fit_content_to_terminal(all_lines)

    def _render_header(self, state: AgentExecutionState) -> str:
        """Rendert den Header."""
        runtime = human_readable_time(time.time() - state.start_time)
        title = self.style.Bold(f"🤖 {state.agent_name}")
        phase = self.style.CYAN(state.execution_phase.upper())
        health_color = self.style.GREEN if state.error_count == 0 else self.style.YELLOW
        health = health_color(f"Fehler: {state.error_count}")

        header_line = f"{title} [{phase}] | {health} | ⏱️ {runtime}"
        separator = self.style.GREY("═" * min(len(remove_styles(header_line)), self._terminal_width - 2))

        return f"{header_line}\n{separator}"

    def _render_outline_section(self, state: AgentExecutionState) -> list:
        """Rendert die Outline-Sektion."""
        outline = state.outline
        progress = state.outline_progress
        if not outline or not outline.get('steps'):
            return []

        lines = [self.style.Bold(self.style.YELLOW("📋 Agenten-Plan"))]

        for i, step in enumerate(outline['steps'][:5], 1):  # Nur erste 5 Schritte
            status_icon = "⏸️"
            line_style = self.style.GREY

            if i < progress['current_step']:
                status_icon = "✅"
                line_style = self.style.GREEN
            elif i == progress['current_step']:
                status_icon = "🔄"
                line_style = self.style.Bold

            desc = step.get('description', f'Schritt {i}')[:60]  # Beschreibung kürzen
            method = self.style.CYAN(f"({step.get('method', 'N/A')})")

            lines.append(line_style(f"  {status_icon} Schritt {i}: {desc} {method}"))

        if len(outline['steps']) > 5:
            lines.append(self.style.GREY(f"  ... und {len(outline['steps']) - 5} weitere Schritte"))

        return lines

    def _render_reasoning_section(self, state: AgentExecutionState) -> list:
        """Rendert die Reasoning-Sektion."""
        notes = state.reasoning_notes
        if not notes:
            return []

        lines = [self.style.Bold(self.style.YELLOW("🧠 Denkprozess"))]

        # Nur die neueste Notiz anzeigen
        note = notes[-1]
        thought = note.get('thought', '...')[:100]  # Gedanken kürzen
        lines.append(f"  💭 {thought}")

        if note.get('current_focus'):
            focus = note['current_focus'][:80]
            lines.append(f"  🎯 Fokus: {self.style.CYAN(focus)}")

        if note.get('confidence_level') is not None:
            confidence = note['confidence_level']
            lines.append(f"  📊 Zuversicht: {self.style.YELLOW(f'{confidence:.0%}')}")

        if note.get('key_insights'):
            lines.append(f"  💡 Erkenntnisse:")
            for insight in note['key_insights'][:2]:  # Nur erste 2 Erkenntnisse
                insight_text = insight[:70]
                lines.append(f"    • {self.style.GREY(insight_text)}")

        return lines

    def _render_activity_section(self, state: AgentExecutionState) -> list:
        """Rendert die aktuelle Aktivität."""
        lines = [self.style.Bold(self.style.YELLOW(f"🔄 Aktivität (Loop {state.current_reasoning_loop})"))]

        if state.active_delegation:
            delegation = state.active_delegation

            if delegation['type'] == 'plan_creation':
                desc = delegation['description'][:80]
                lines.append(f"  📝 {desc}")

                if delegation.get('goals'):
                    lines.append(f"  🎯 Ziele: {len(delegation['goals'])}")
                    for goal in delegation['goals'][:2]:  # Nur erste 2 Ziele
                        goal_text = goal[:60]
                        lines.append(f"    • {self.style.GREY(goal_text)}")

            elif delegation['type'] == 'tool_delegation':
                desc = delegation['description'][:80]
                lines.append(f"  🛠️ {desc}")
                status = delegation.get('status', 'unbekannt')
                lines.append(f"  📊 Status: {self.style.CYAN(status)}")

                if delegation.get('tools'):
                    tools_text = ', '.join(delegation['tools'][:3])  # Nur erste 3 Tools
                    lines.append(f"  🔧 Tools: {tools_text}")

        # LLM-Statistiken kompakt
        llm = state.llm_interactions
        if llm['total_calls'] > 0:
            cost = f"${llm['total_cost']:.3f}"
            lines.append(
                self.style.GREY(f"  🤖 LLM: {llm['total_calls']} Calls | {cost} | {llm['total_tokens']:,} Tokens"))

        # LLM Stream (gekürzt)
        if self.llm_stream_chunks:
            stream_lines = self.llm_stream_chunks.splitlines()[-8:]
            for stream_line in stream_lines:
                truncated = stream_line[:self._terminal_width - 6]
                lines.append(self.style.GREY(f"  💬 {truncated}"))

        return lines

    def _render_task_plan_section(self, state: AgentExecutionState) -> list:
        """Rendert den Task-Plan kompakt."""
        plan: TaskPlan = state.active_task_plan
        if not plan:
            return []

        lines = [self.style.Bold(self.style.YELLOW(f"⚙️ Plan: {plan.name}"))]

        # Nur aktive und wichtige Tasks anzeigen
        sorted_tasks = sorted(plan.tasks, key=lambda t: (
            0 if t.status == 'running' else
            1 if t.status == 'failed' else
            2 if t.status == 'pending' else 3,
            getattr(t, 'priority', 99),
            t.id
        ))

        displayed_count = 0
        max_display = 5

        for task in sorted_tasks:
            if displayed_count >= max_display:
                remaining = len(sorted_tasks) - displayed_count
                lines.append(self.style.GREY(f"  ... und {remaining} weitere Tasks"))
                break

            icon = {"pending": "⏳", "running": "🔄", "completed": "✅", "failed": "❌"}.get(task.status, "❓")
            style_func = {"pending": self.style.GREY, "running": self.style.WHITE,
                          "completed": self.style.GREEN, "failed": self.style.RED}.get(task.status, self.style.WHITE)

            desc = task.description[:50]  # Beschreibung kürzen
            lines.append(style_func(f"  {icon} {task.id}: {desc}"))

            # Fehler anzeigen wenn vorhanden
            if hasattr(task, 'error') and task.error:
                error_text = task.error[:self._terminal_width - 5]
                lines.append(self.style.RED(f"    🔥 {error_text}"))

            displayed_count += 1

        return lines

    def _render_tool_history_section(self, state: AgentExecutionState) -> list:
        """Rendert die Tool-Historie kompakt."""
        history = state.tool_history
        if not history:
            return []

        lines = [self.style.Bold(self.style.YELLOW("🛠️ Tool-Historie"))]

        # Nur die letzten 5 Tools
        for event in reversed(history[-5:]):
            icon = "✅" if event.success else "❌"
            style_func = self.style.GREEN if event.success else self.style.RED
            duration = f"({human_readable_time(event.node_duration)})" if event.node_duration else ""

            tool_line = f"  {icon} {event.tool_name} {duration} {arguments_summary(event.tool_args, self._terminal_width)}"
            lines.append(style_func(tool_line))

            # Fehler kurz anzeigen
            if not event.success and event.tool_error:
                error_text = event.tool_error[:self._terminal_width - 5]
                lines.append(self.style.RED(f"    💥 {error_text}"))

        return lines

    def _render_system_flow_section(self, state: AgentExecutionState) -> list:
        """Rendert den System-Flow kompakt."""
        if not state.node_flow:
            return []

        lines = [self.style.Bold(self.style.YELLOW("🔧 System-Ablauf"))]

        # Nur aktive Nodes und die letzten paar
        recent_nodes = state.node_flow[-4:]  # Letzte 4 Nodes

        for i, node_name in enumerate(recent_nodes):
            is_last = (i == len(recent_nodes) - 1)
            prefix = "└─" if is_last else "├─"
            is_active = node_name in state.active_nodes
            icon = "🔄" if is_active else "✅"
            style_func = self.style.Bold if is_active else self.style.GREEN

            node_display = node_name[:30]  # Node-Namen kürzen
            lines.append(style_func(f"  {prefix} {icon} {node_display}"))

            # Aktive Node Details
            if is_active:
                last_event = state.last_event_per_node.get(node_name)
                if last_event and last_event.event_type == 'tool_call' and last_event.status == NodeStatus.RUNNING:
                    tool_name = last_event.tool_name[:25]
                    child_prefix = "     " if is_last else "  │  "
                    lines.append(self.style.GREY(f"{child_prefix}🔧 {tool_name}"))

        if len(state.node_flow) > 4:
            lines.append(self.style.GREY(f"  ... und {len(state.node_flow) - 4} weitere Nodes"))

        return lines

    def print_final_summary(self):
        """Zeigt die finale Zusammenfassung."""
        self._update_terminal_size()  # Terminal-Größe neu ermitteln
        output_lines = self._render_full_display()
        print('\n'.join(output_lines))
        summary_lines = [
            "",
            self.style.GREEN2(self.style.Bold("🏁 Ausführung Abgeschlossen")),
            self.style.GREY(f"Events verarbeitet: {self.processor.state.event_count}"),
            self.style.GREY(f"Gesamtlaufzeit: {human_readable_time(time.time() - self.processor.state.start_time)}"),
            ""
        ]

        for line in summary_lines:
            print(line)

clear() staticmethod

Speichert aktuellen Terminal-Content und cleared das Terminal Systemagnostisch (Windows/Unix)

Source code in toolboxv2/mods/isaa/extras/terminal_progress.py

@staticmethod
def clear():
    """
    Speichert aktuellen Terminal-Content und cleared das Terminal
    Systemagnostisch (Windows/Unix)
    """
    # Clear terminal - systemagnostisch
    if os.name == 'nt':  # Windows
        os.system('cls')
    else:  # Unix/Linux/macOS
        os.system('clear')

live_print(*args, **kwargs)

Live print ohne Content-Speicherung für temporäre Ausgaben

Source code in toolboxv2/mods/isaa/extras/terminal_progress.py

def live_print(self,*args, **kwargs):
    """
    Live print ohne Content-Speicherung für temporäre Ausgaben
    """
    self._original_print(*args, **kwargs)

print(*args, **kwargs)

Überladene print Funktion die automatisch Content speichert

Source code in toolboxv2/mods/isaa/extras/terminal_progress.py

def print(self, *args, **kwargs):
    """
    Überladene print Funktion die automatisch Content speichert
    """
    # Capture output in StringIO für Effizienz
    output = StringIO()
    if 'file' in kwargs:
        del kwargs['file']
    self._original_print(*args, file=output, **kwargs)
    content = output.getvalue()

    # Speichere nur wenn content nicht leer
    if content.strip():
        self._terminal_content.append(content.rstrip('\n'))

    # Normale Ausgabe
    self._original_print(*args, **kwargs)

print_final_summary()

Zeigt die finale Zusammenfassung.

Source code in toolboxv2/mods/isaa/extras/terminal_progress.py

def print_final_summary(self):
    """Zeigt die finale Zusammenfassung."""
    self._update_terminal_size()  # Terminal-Größe neu ermitteln
    output_lines = self._render_full_display()
    print('\n'.join(output_lines))
    summary_lines = [
        "",
        self.style.GREEN2(self.style.Bold("🏁 Ausführung Abgeschlossen")),
        self.style.GREY(f"Events verarbeitet: {self.processor.state.event_count}"),
        self.style.GREY(f"Gesamtlaufzeit: {human_readable_time(time.time() - self.processor.state.start_time)}"),
        ""
    ]

    for line in summary_lines:
        print(line)

progress_callback(event) async

Haupteingangspunkt für Progress Events.

Source code in toolboxv2/mods/isaa/extras/terminal_progress.py

async def progress_callback(self, event: ProgressEvent):
    """Haupteingangspunkt für Progress Events."""
    if event.event_type == 'execution_start':
        self.processor = StateProcessor()
        self._is_initialized = True


    self.processor.process_event(event)

    # LLM Stream Handling
    if event.event_type == 'llm_stream_chunk':
        self.llm_stream_chunks += event.llm_output
        # Stream-Chunks auf vernünftige Größe begrenzen
        lines = self.llm_stream_chunks.replace('\\n', '\n').split('\n')
        if len(lines) > 8:
            self.llm_stream_chunks = '\n'.join(lines[-8:])
        self.buffer += 1
        if self.buffer > 5:
            self.buffer = 0
        else:
            return

    if event.event_type == 'llm_call':
        self.llm_stream_chunks = ""

    # Display nur bei wichtigen Events oder zeitbasiert aktualisieren
    should_update = (
        time.time() - self._last_update_time > self._display_interval or
        event.event_type in ['execution_complete', 'outline_created', 'plan_created', 'node_enter']
    )

    if should_update and self._is_initialized:
        self._update_display()
        self._last_update_time = time.time()


    if event.event_type in ['execution_complete', 'error']:
        self.restore_content()
        self.print_final_summary()

restore_content()

Stellt den gespeicherten Terminal-Content in einer Aktion wieder her Effizient durch join operation

Source code in toolboxv2/mods/isaa/extras/terminal_progress.py

def restore_content(self):
    """
    Stellt den gespeicherten Terminal-Content in einer Aktion wieder her
    Effizient durch join operation
    """
    if self._terminal_content:
        # Effiziente Wiederherstellung mit join
        restored_output = '\n'.join(self._terminal_content)
        self._original_print(restored_output)

StateProcessor

Verarbeitet ProgressEvents und aktualisiert den AgentExecutionState.

Source code in toolboxv2/mods/isaa/extras/terminal_progress.py

class StateProcessor:
    """Verarbeitet ProgressEvents und aktualisiert den AgentExecutionState."""

    def __init__(self):
        self.state = AgentExecutionState()

    def process_event(self, event: ProgressEvent):
        self.state.event_count += 1
        if event.agent_name:
            self.state.agent_name = event.agent_name

        # System-Level Events
        if event.event_type == 'node_enter' and event.node_name:
            self.state.active_nodes.add(event.node_name)
            if event.node_name not in self.state.node_flow:
                self.state.node_flow.append(event.node_name)
        elif event.event_type == 'node_exit' and event.node_name:
            self.state.active_nodes.discard(event.node_name)
        elif event.event_type == 'error':
            self.state.error_count += 1

        if event.node_name:
            self.state.last_event_per_node[event.node_name] = event

        # Outline & Reasoning Events
        if event.event_type == 'outline_created' and isinstance(event.metadata.get('outline'), dict):
            self.state.execution_phase = 'planning'
            self.state.outline = event.metadata['outline']
            self.state.outline_progress['total_steps'] = len(self.state.outline.get('steps', []))

        elif event.event_type == 'reasoning_loop':
            self.state.execution_phase = 'reasoning'
            self.state.current_reasoning_loop = event.metadata.get('loop_number', 0)
            self.state.outline_progress['current_step'] = event.metadata.get('outline_step', 0) + 1
            self.state.active_delegation = None

        # Task Plan & Execution Events
        elif event.event_type == 'plan_created' and event.metadata.get('full_plan'):
            self.state.execution_phase = 'executing_plan'
            self.state.active_task_plan = event.metadata['full_plan']
            self.state.active_delegation = None

        elif event.event_type in ['task_start', 'task_complete', 'task_error']:
            self._update_task_plan_status(event)

        # Tool & LLM Events
        elif event.event_type == 'tool_call':
            if event.is_meta_tool:
                self._process_meta_tool_call(event)
            else:
                if event.status in [NodeStatus.COMPLETED, NodeStatus.FAILED]:
                    self.state.tool_history.append(event)
                    if len(self.state.tool_history) > 5:
                        self.state.tool_history.pop(0)

        elif event.event_type == 'llm_call' and event.success:
            llm = self.state.llm_interactions
            llm['total_calls'] += 1
            llm['total_cost'] += event.llm_cost or 0
            llm['total_tokens'] += event.llm_total_tokens or 0

        elif event.event_type == 'execution_complete':
            self.state.execution_phase = 'completed'

    def _process_meta_tool_call(self, event: ProgressEvent):
        args = event.tool_args or {}
        if event.status != NodeStatus.RUNNING:
            return

        if event.tool_name == 'internal_reasoning':
            note = {k: args.get(k) for k in ['thought', 'current_focus', 'key_insights', 'confidence_level']}
            self.state.reasoning_notes.append(note)
            if len(self.state.reasoning_notes) > 3:
                self.state.reasoning_notes.pop(0)

        elif event.tool_name == 'delegate_to_llm_tool_node':
            self.state.active_delegation = {
                'type': 'tool_delegation',
                'description': args.get('task_description', 'N/A'),
                'tools': args.get('tools_list', []),
                'status': 'running'
            }

        elif event.tool_name == 'create_and_execute_plan':
            self.state.active_delegation = {
                'type': 'plan_creation',
                'description': f"Erstelle Plan für {len(args.get('goals', []))} Ziele",
                'goals': args.get('goals', []),
                'status': 'planning'
            }

    def _update_task_plan_status(self, event: ProgressEvent):
        plan = self.state.active_task_plan
        if not plan or not hasattr(plan, 'tasks'):
            return

        for task in plan.tasks:
            if hasattr(task, 'id') and task.id == event.task_id:
                if event.event_type == 'task_start':
                    task.status = 'running'
                elif event.event_type == 'task_complete':
                    task.status = 'completed'
                    task.result = event.tool_result or (event.metadata or {}).get("result")
                elif event.event_type == 'task_error':
                    task.status = 'failed'
                    task.error = (event.error_details or {}).get('message', 'Unbekannter Fehler')
                break

arguments_summary(tool_args, max_length=50)

Creates a summary of the tool arguments for display purposes.

Parameters:

Name	Type	Description	Default
tool_args	dict[str, Any]	Dictionary containing tool arguments	required
max_length	int	Maximum length for individual argument values in summary	50

Returns:

Type	Description
str	Formatted string summary of the arguments

Source code in toolboxv2/mods/isaa/extras/terminal_progress.py

def arguments_summary(tool_args: dict[str, Any], max_length: int = 50) -> str:
    """
    Creates a summary of the tool arguments for display purposes.

    Args:
        tool_args: Dictionary containing tool arguments
        max_length: Maximum length for individual argument values in summary

    Returns:
        Formatted string summary of the arguments
    """

    if not tool_args:
        return "No arguments"

    return_str = ""

    # Handle different types of arguments
    for key, value in tool_args.items():
        # Format the key
        formatted_key = key.replace('_', ' ').title()

        # Handle different value types
        if value is None:
            formatted_value = "None"
        elif isinstance(value, bool):
            formatted_value = str(value)
        elif isinstance(value, (int, float)):
            formatted_value = str(value)
        elif isinstance(value, str):
            # Truncate long strings
            if len(value) > max_length:
                formatted_value = f'"{value[:max_length - 3]}..."'
            else:
                formatted_value = f'"{value}"'
        elif isinstance(value, list):
            if not value:
                formatted_value = "[]"
            elif len(value) == 1:
                item = value[0]
                if isinstance(item, str) and len(item) > max_length:
                    formatted_value = f'["{item[:max_length - 6]}..."]'
                else:
                    formatted_value = f'["{item}"]' if isinstance(item, str) else f'[{item}]'
            else:
                formatted_value = f"[{len(value)} items]"
        elif isinstance(value, dict):
            if not value:
                formatted_value = "{}"
            else:
                keys = list(value.keys())[:3]  # Show first 3 keys
                if len(value) <= 3:
                    formatted_value = f"{{{', '.join(keys)}}}"
                else:
                    formatted_value = f"{{{', '.join(keys)}, ...}} ({len(value)} keys)"
        else:
            # Fallback for other types
            str_value = str(value)
            if len(str_value) > max_length:
                formatted_value = f"{str_value[:max_length - 3]}..."
            else:
                formatted_value = str_value

        # Add to return string
        if return_str:
            return_str += ", "
        return_str += f"{formatted_key}: {formatted_value}"

    # Handle meta-tool specific summaries
    if "tool_name" in tool_args:
        tool_name = tool_args["tool_name"]

        if tool_name == "internal_reasoning":
            meta_summary = []
            if "thought_number" in tool_args and "total_thoughts" in tool_args:
                meta_summary.append(f"Thought {tool_args['thought_number']}/{tool_args['total_thoughts']}")
            if "current_focus" in tool_args and tool_args["current_focus"]:
                focus = tool_args["current_focus"]
                if len(focus) > 30:
                    focus = focus[:27] + "..."
                meta_summary.append(f"Focus: {focus}")
            if "confidence_level" in tool_args:
                meta_summary.append(f"Confidence: {tool_args['confidence_level']}")

            if meta_summary:
                return_str = f"Internal Reasoning - {', '.join(meta_summary)}"

        elif tool_name == "manage_internal_task_stack":
            action = tool_args.get("action", "unknown")
            task_desc = tool_args.get("task_description", "")
            if len(task_desc) > 40:
                task_desc = task_desc[:37] + "..."
            return_str = f"Task Stack - Action: {action.title()}, Task: {task_desc}"

        elif tool_name == "delegate_to_llm_tool_node":
            task_desc = tool_args.get("task_description", "")
            tools_count = len(tool_args.get("tools_list", []))
            if len(task_desc) > 40:
                task_desc = task_desc[:37] + "..."
            return_str = f"Delegate - Task: {task_desc}, Tools: {tools_count}"

        elif tool_name == "create_and_execute_plan":
            goals_count = len(tool_args.get("goals", []))
            return_str = f"Create Plan - Goals: {goals_count}"

        elif tool_name == "advance_outline_step":
            completed = tool_args.get("step_completed", False)
            next_focus = tool_args.get("next_step_focus", "")
            if len(next_focus) > 30:
                next_focus = next_focus[:27] + "..."
            return_str = f"Advance Step - Completed: {completed}, Next: {next_focus}"

        elif tool_name == "write_to_variables":
            scope = tool_args.get("scope", "unknown")
            key = tool_args.get("key", "")
            return_str = f"Write Variable - Scope: {scope}, Key: {key}"

        elif tool_name == "read_from_variables":
            scope = tool_args.get("scope", "unknown")
            key = tool_args.get("key", "")
            return_str = f"Read Variable - Scope: {scope}, Key: {key}"

        elif tool_name == "direct_response":
            final_answer = tool_args.get("final_answer", "")
            if len(final_answer) > 50:
                final_answer = final_answer[:47] + "..."
            return_str = f"Direct Response - Answer: {final_answer}"

    # Handle live tool specific summaries
    elif any(key in tool_args for key in ["code", "filepath", "package_name"]):
        if "code" in tool_args:
            code = tool_args["code"]
            code_preview = code.replace('\n', ' ').strip()
            if len(code_preview) > 40:
                code_preview = code_preview[:37] + "..."
            return_str = f"Execute Code - {code_preview}"

        elif "filepath" in tool_args:
            filepath = tool_args["filepath"]
            if "content" in tool_args:
                content_length = len(str(tool_args["content"]))
                return_str = f"File Operation - Path: {filepath}, Content: {content_length} chars"
            elif "old_content" in tool_args and "new_content" in tool_args:
                return_str = f"Replace in File - Path: {filepath}, Replace operation"
            else:
                return_str = f"File Operation - Path: {filepath}"

        elif "package_name" in tool_args:
            package = tool_args["package_name"]
            version = tool_args.get("version", "latest")
            return_str = f"Install Package - {package} ({version})"

    # Ensure we don't exceed reasonable length for the entire summary
    if len(return_str) > 200:
        return_str = return_str[:197] + "..."

    return return_str

human_readable_time(seconds)

Konvertiert Sekunden in ein menschlich lesbares Format.

Source code in toolboxv2/mods/isaa/extras/terminal_progress.py

def human_readable_time(seconds: float) -> str:
    """Konvertiert Sekunden in ein menschlich lesbares Format."""
    if seconds is None:
        return ""
    if seconds < 1:
        return f"{seconds * 1000:.0f}ms"
    seconds = int(seconds)
    if seconds < 60:
        return f"{seconds}s"
    minutes, seconds = divmod(seconds, 60)
    if minutes < 60:
        return f"{minutes}m {seconds}s"
    hours, minutes = divmod(minutes, 60)
    return f"{hours}h {minutes}m"

verbose_output

DynamicVerboseFormatter

Unified, dynamic formatter that adapts to screen size

Source code in toolboxv2/mods/isaa/extras/verbose_output.py

class DynamicVerboseFormatter:
    """Unified, dynamic formatter that adapts to screen size"""

    def __init__(self, print_func=None, min_width: int = 40, max_width: int = 240):
        self.style = Style()
        self.print = print_func or print
        self.min_width = min_width
        self.max_width = max_width
        self._terminal_width = self._get_terminal_width()


    def get_git_info(self):
        """Checks for a git repo and returns its name and branch, or None."""
        try:
            # Check if we are in a git repository
            subprocess.check_output(['git', 'rev-parse', '--is-inside-work-tree'], stderr=subprocess.DEVNULL)

            # Get the repo name (root folder name)
            repo_root = subprocess.check_output(['git', 'rev-parse', '--show-toplevel'],
                                                stderr=subprocess.DEVNULL).strip().decode('utf-8')
            repo_name = os.path.basename(repo_root)

            # Get the current branch name
            branch = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD'],
                                             stderr=subprocess.DEVNULL).strip().decode('utf-8')

            return repo_name, branch
        except (subprocess.CalledProcessError, FileNotFoundError):
            # This handles cases where 'git' is not installed or it's not a git repo
            return None

    def _get_terminal_width(self) -> int:
        """Get current terminal width with fallback"""
        try:
            width = shutil.get_terminal_size().columns
            return max(self.min_width, min(width - 2, self.max_width))
        except (OSError, AttributeError):
            return 80

    def _wrap_text(self, text: str, width: int = None) -> list[str]:
        """Wrap text to fit terminal width"""
        if width is None:
            width = self._terminal_width - 4  # Account for borders

        words = text.split()
        lines = []
        current_line = []
        current_length = 0

        for word in words:
            if current_length + len(word) + len(current_line) <= width:
                current_line.append(word)
                current_length += len(word)
            else:
                if current_line:
                    lines.append(' '.join(current_line))
                current_line = [word]
                current_length = len(word)

        if current_line:
            lines.append(' '.join(current_line))

        return lines

    def _create_border(self, char: str = "─", width: int = None) -> str:
        """Create a border line that fits the terminal"""
        if width is None:
            width = self._terminal_width
        return char * width

    def _center_text(self, text: str, width: int = None) -> str:
        """Center text within the given width"""
        if width is None:
            width = self._terminal_width

        # Remove ANSI codes for length calculation
        clean_text = self._strip_ansi(text)
        padding = max(0, (width - len(clean_text)) // 2)
        return " " * padding + text

    def _strip_ansi(self, text: str) -> str:
        """Remove ANSI escape codes for length calculation"""
        import re
        ansi_escape = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])')
        return ansi_escape.sub('', text)

    def print_header(self, text: str):
        """Print a dynamic header that adapts to screen size"""
        self._terminal_width = self._get_terminal_width()

        if self._terminal_width < 60:  # Tiny screen
            self.print()
            self.print(self.style.CYAN("=" * self._terminal_width))
            self.print(self.style.CYAN(self.style.Bold(text)))
            self.print(self.style.CYAN("=" * self._terminal_width))
        else:  # Regular/large screen
            border_width = min(len(text) + 2, self._terminal_width - 2)
            border = "─" * border_width

            self.print()
            self.print(self.style.CYAN(f"┌{border}┐"))
            self.print(self.style.CYAN(f"│ {self.style.Bold(text).center(border_width - 2)} │"))
            self.print(self.style.CYAN(f"└{border}┘"))
        self.print()

    def print_section(self, title: str, content: str):
        """Print a clean section with adaptive formatting"""
        self._terminal_width = self._get_terminal_width()

        # Title
        if self._terminal_width < 60:
            self.print(f"\n{self.style.BLUE('●')} {self.style.Bold(title)}")
        else:
            self.print(f"\n{self.style.BLUE('●')} {self.style.Bold(self.style.BLUE(title))}")

        # Content with proper wrapping
        for line in content.split('\n'):
            if line.strip():
                wrapped_lines = self._wrap_text(line.strip())
                for wrapped_line in wrapped_lines:
                    if self._terminal_width < 60:
                        self.print(f"  {wrapped_line}")
                    else:
                        self.print(f"  {self.style.GREY('│')} {wrapped_line}")
        self.print()

    def print_progress_bar(self, current: int, maximum: int, title: str = "Progress"):
        """Dynamic progress bar that adapts to screen size"""
        self._terminal_width = self._get_terminal_width()

        # Calculate bar width based on screen size
        if self._terminal_width < 60:
            bar_width = 10
            template = f"\r{title}: [{{}}] {current}/{maximum}"
        else:
            bar_width = min(30, self._terminal_width - 30)
            template = f"\r{self.style.CYAN(title)}: [{{}}] {current}/{maximum} ({current / maximum * 100:.1f}%)"

        progress = int((current / maximum) * bar_width)
        bar = "█" * progress + "░" * (bar_width - progress)

        self.print(template.format(bar), end='', flush=True)

    def print_state(self, state: str, details: dict[str, Any] = None) -> str:
        """Print current state with adaptive formatting"""
        self._terminal_width = self._get_terminal_width()

        state_colors = {
            'ACTION': self.style.GREEN2,
            'PROCESSING': self.style.YELLOW2,
            'BRAKE': self.style.RED2,
            'DONE': self.style.BLUE2,
            'ERROR': self.style.RED,
            'SUCCESS': self.style.GREEN,
            'INFO': self.style.CYAN
        }

        color_func = state_colors.get(state.upper(), self.style.WHITE2)

        if self._terminal_width < 60:
            # Compact format for small screens
            self.print(f"\n[{color_func(state)}]")
            result = f"\n[{state}]"
        else:
            # Full format for larger screens
            self.print(f"\n{self.style.Bold('State:')} {color_func(state)}")
            result = f"\nState: {state}"

        if details:
            for key, value in details.items():
                # Truncate long values on small screens
                if self._terminal_width < 60 and len(str(value)) > 30:
                    display_value = str(value)[:27] + "..."
                else:
                    display_value = str(value)

                if self._terminal_width < 60:
                    self.print(f"  {key}: {display_value}")
                    result += f"\n  {key}: {display_value}"
                else:
                    self.print(f"  {self.style.GREY('├─')} {self.style.CYAN(key)}: {display_value}")
                    result += f"\n  ├─ {key}: {display_value}"

        return result

    def print_code_block(self, code: str, language: str = "python"):
        """Print code with syntax awareness and proper formatting"""
        self._terminal_width = self._get_terminal_width()

        if self._terminal_width < 60:
            # Simple format for small screens
            self.print(f"\n{self.style.GREY('Code:')}")
            for line in code.split('\n'):
                self.print(f"  {line}")
        else:
            # Detailed format for larger screens
            self.print(f"\n{self.style.BLUE('┌─')} {self.style.YELLOW2(f'{language.upper()} Code')}")

            lines = code.split('\n')
            for i, line in enumerate(lines):
                if i == len(lines) - 1 and not line.strip():
                    continue

                # Wrap long lines
                if len(line) > self._terminal_width - 6:
                    wrapped = self._wrap_text(line, self._terminal_width - 6)
                    for j, wrapped_line in enumerate(wrapped):
                        prefix = "│" if j == 0 else "│"
                        self.print(f"{self.style.BLUE(prefix)} {wrapped_line}")
                else:
                    self.print(f"{self.style.BLUE('│')} {line}")

            self.print(f"{self.style.BLUE('└─')} {self.style.GREY('End of code block')}")

    def print_table(self, headers: list[str], rows: list[list[str]]):
        """Print a dynamic table that adapts to screen size"""
        self._terminal_width = self._get_terminal_width()

        if not rows:
            return

        # Calculate column widths
        all_data = [headers] + rows
        col_widths = []

        for col in range(len(headers)):
            max_width = max(len(str(row[col])) for row in all_data if col < len(row))
            col_widths.append(min(max_width, self._terminal_width // len(headers) - 2))

        # Adjust if total width exceeds terminal
        total_width = sum(col_widths) + len(headers) * 3 + 1
        if total_width > self._terminal_width:
            # Proportionally reduce column widths
            scale_factor = (self._terminal_width - len(headers) * 3 - 1) / sum(col_widths)
            col_widths = [max(8, int(w * scale_factor)) for w in col_widths]

        # Print table
        self._print_table_row(headers, col_widths, is_header=True)
        self._print_table_separator(col_widths)

        for row in rows:
            self._print_table_row(row, col_widths)

    def _print_table_row(self, row: list[str], widths: list[int], is_header: bool = False):
        """Helper method to print a table row"""
        formatted_cells = []
        for _i, (cell, width) in enumerate(zip(row, widths, strict=False)):
            cell_str = str(cell)
            if len(cell_str) > width:
                cell_str = cell_str[:width - 3] + "..."

            if is_header:
                formatted_cells.append(self.style.Bold(self.style.CYAN(cell_str.ljust(width))))
            else:
                formatted_cells.append(cell_str.ljust(width))

        self.print(f"│ {' │ '.join(formatted_cells)} │")

    def _print_table_separator(self, widths: list[int]):
        """Helper method to print table separator"""
        parts = ['─' * w for w in widths]
        self.print(f"├─{'─┼─'.join(parts)}─┤")

    async def process_with_spinner(self, message: str, coroutine):
        """Execute coroutine with adaptive spinner"""
        self._terminal_width = self._get_terminal_width()

        if self._terminal_width < 60:
            # Simple spinner for small screens
            spinner_symbols = "⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏"
        else:
            # Detailed spinner for larger screens
            spinner_symbols = "⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏"

        # Truncate message if too long
        if len(message) > self._terminal_width - 10:
            display_message = message[:self._terminal_width - 13] + "..."
        else:
            display_message = message

        with Spinner(f"{self.style.CYAN('●')} {display_message}", symbols=spinner_symbols):
            return await coroutine

    def print_git_info(self) -> str | None:
        """Get current git branch with error handling"""
        try:
            result = subprocess.run(
                ['git', 'rev-parse', '--abbrev-ref', 'HEAD'],
                capture_output=True, text=True, timeout=2
            )
            if result.returncode == 0 and result.stdout.strip():
                branch = result.stdout.strip()

                # Check for uncommitted changes
                status_result = subprocess.run(
                    ['git', 'status', '--porcelain'],
                    capture_output=True, text=True, timeout=1
                )
                dirty = "*" if status_result.stdout.strip() else ""

                git_info = f"{branch}{dirty}"
                self.print_info(f"Git: {git_info}")
                return git_info
        except (subprocess.TimeoutExpired, FileNotFoundError, subprocess.SubprocessError):
            pass
        return None

    # Convenience methods with consistent styling
    def print_error(self, message: str):
        """Print error message with consistent formatting"""
        self.print(f"{self.style.RED('✗')} {self.style.RED(message)}")

    def print_success(self, message: str):
        """Print success message with consistent formatting"""
        self.print(f"{self.style.GREEN('✓')} {self.style.GREEN(message)}")

    def print_warning(self, message: str):
        """Print warning message with consistent formatting"""
        self.print(f"{self.style.YELLOW('⚠')} {self.style.YELLOW(message)}")

    def print_info(self, message: str):
        """Print info message with consistent formatting"""
        self.print(f"{self.style.CYAN('ℹ')} {self.style.CYAN(message)}")

    def print_debug(self, message: str):
        """Print debug message with consistent formatting"""
        self.print(f"{self.style.GREY('🐛')} {self.style.GREY(message)}")

get_git_info()

Checks for a git repo and returns its name and branch, or None.

Source code in toolboxv2/mods/isaa/extras/verbose_output.py

def get_git_info(self):
    """Checks for a git repo and returns its name and branch, or None."""
    try:
        # Check if we are in a git repository
        subprocess.check_output(['git', 'rev-parse', '--is-inside-work-tree'], stderr=subprocess.DEVNULL)

        # Get the repo name (root folder name)
        repo_root = subprocess.check_output(['git', 'rev-parse', '--show-toplevel'],
                                            stderr=subprocess.DEVNULL).strip().decode('utf-8')
        repo_name = os.path.basename(repo_root)

        # Get the current branch name
        branch = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD'],
                                         stderr=subprocess.DEVNULL).strip().decode('utf-8')

        return repo_name, branch
    except (subprocess.CalledProcessError, FileNotFoundError):
        # This handles cases where 'git' is not installed or it's not a git repo
        return None

print_code_block(code, language='python')

Print code with syntax awareness and proper formatting

Source code in toolboxv2/mods/isaa/extras/verbose_output.py

def print_code_block(self, code: str, language: str = "python"):
    """Print code with syntax awareness and proper formatting"""
    self._terminal_width = self._get_terminal_width()

    if self._terminal_width < 60:
        # Simple format for small screens
        self.print(f"\n{self.style.GREY('Code:')}")
        for line in code.split('\n'):
            self.print(f"  {line}")
    else:
        # Detailed format for larger screens
        self.print(f"\n{self.style.BLUE('┌─')} {self.style.YELLOW2(f'{language.upper()} Code')}")

        lines = code.split('\n')
        for i, line in enumerate(lines):
            if i == len(lines) - 1 and not line.strip():
                continue

            # Wrap long lines
            if len(line) > self._terminal_width - 6:
                wrapped = self._wrap_text(line, self._terminal_width - 6)
                for j, wrapped_line in enumerate(wrapped):
                    prefix = "│" if j == 0 else "│"
                    self.print(f"{self.style.BLUE(prefix)} {wrapped_line}")
            else:
                self.print(f"{self.style.BLUE('│')} {line}")

        self.print(f"{self.style.BLUE('└─')} {self.style.GREY('End of code block')}")

print_debug(message)

Print debug message with consistent formatting

Source code in toolboxv2/mods/isaa/extras/verbose_output.py

def print_debug(self, message: str):
    """Print debug message with consistent formatting"""
    self.print(f"{self.style.GREY('🐛')} {self.style.GREY(message)}")

print_error(message)

Print error message with consistent formatting

Source code in toolboxv2/mods/isaa/extras/verbose_output.py

def print_error(self, message: str):
    """Print error message with consistent formatting"""
    self.print(f"{self.style.RED('✗')} {self.style.RED(message)}")

print_git_info()

Get current git branch with error handling

Source code in toolboxv2/mods/isaa/extras/verbose_output.py

def print_git_info(self) -> str | None:
    """Get current git branch with error handling"""
    try:
        result = subprocess.run(
            ['git', 'rev-parse', '--abbrev-ref', 'HEAD'],
            capture_output=True, text=True, timeout=2
        )
        if result.returncode == 0 and result.stdout.strip():
            branch = result.stdout.strip()

            # Check for uncommitted changes
            status_result = subprocess.run(
                ['git', 'status', '--porcelain'],
                capture_output=True, text=True, timeout=1
            )
            dirty = "*" if status_result.stdout.strip() else ""

            git_info = f"{branch}{dirty}"
            self.print_info(f"Git: {git_info}")
            return git_info
    except (subprocess.TimeoutExpired, FileNotFoundError, subprocess.SubprocessError):
        pass
    return None

print_header(text)

Print a dynamic header that adapts to screen size

Source code in toolboxv2/mods/isaa/extras/verbose_output.py

def print_header(self, text: str):
    """Print a dynamic header that adapts to screen size"""
    self._terminal_width = self._get_terminal_width()

    if self._terminal_width < 60:  # Tiny screen
        self.print()
        self.print(self.style.CYAN("=" * self._terminal_width))
        self.print(self.style.CYAN(self.style.Bold(text)))
        self.print(self.style.CYAN("=" * self._terminal_width))
    else:  # Regular/large screen
        border_width = min(len(text) + 2, self._terminal_width - 2)
        border = "─" * border_width

        self.print()
        self.print(self.style.CYAN(f"┌{border}┐"))
        self.print(self.style.CYAN(f"│ {self.style.Bold(text).center(border_width - 2)} │"))
        self.print(self.style.CYAN(f"└{border}┘"))
    self.print()

print_info(message)

Print info message with consistent formatting

Source code in toolboxv2/mods/isaa/extras/verbose_output.py

def print_info(self, message: str):
    """Print info message with consistent formatting"""
    self.print(f"{self.style.CYAN('ℹ')} {self.style.CYAN(message)}")

print_progress_bar(current, maximum, title='Progress')

Dynamic progress bar that adapts to screen size

Source code in toolboxv2/mods/isaa/extras/verbose_output.py

def print_progress_bar(self, current: int, maximum: int, title: str = "Progress"):
    """Dynamic progress bar that adapts to screen size"""
    self._terminal_width = self._get_terminal_width()

    # Calculate bar width based on screen size
    if self._terminal_width < 60:
        bar_width = 10
        template = f"\r{title}: [{{}}] {current}/{maximum}"
    else:
        bar_width = min(30, self._terminal_width - 30)
        template = f"\r{self.style.CYAN(title)}: [{{}}] {current}/{maximum} ({current / maximum * 100:.1f}%)"

    progress = int((current / maximum) * bar_width)
    bar = "█" * progress + "░" * (bar_width - progress)

    self.print(template.format(bar), end='', flush=True)

print_section(title, content)

Print a clean section with adaptive formatting

Source code in toolboxv2/mods/isaa/extras/verbose_output.py

def print_section(self, title: str, content: str):
    """Print a clean section with adaptive formatting"""
    self._terminal_width = self._get_terminal_width()

    # Title
    if self._terminal_width < 60:
        self.print(f"\n{self.style.BLUE('●')} {self.style.Bold(title)}")
    else:
        self.print(f"\n{self.style.BLUE('●')} {self.style.Bold(self.style.BLUE(title))}")

    # Content with proper wrapping
    for line in content.split('\n'):
        if line.strip():
            wrapped_lines = self._wrap_text(line.strip())
            for wrapped_line in wrapped_lines:
                if self._terminal_width < 60:
                    self.print(f"  {wrapped_line}")
                else:
                    self.print(f"  {self.style.GREY('│')} {wrapped_line}")
    self.print()

print_state(state, details=None)

Print current state with adaptive formatting

Source code in toolboxv2/mods/isaa/extras/verbose_output.py

def print_state(self, state: str, details: dict[str, Any] = None) -> str:
    """Print current state with adaptive formatting"""
    self._terminal_width = self._get_terminal_width()

    state_colors = {
        'ACTION': self.style.GREEN2,
        'PROCESSING': self.style.YELLOW2,
        'BRAKE': self.style.RED2,
        'DONE': self.style.BLUE2,
        'ERROR': self.style.RED,
        'SUCCESS': self.style.GREEN,
        'INFO': self.style.CYAN
    }

    color_func = state_colors.get(state.upper(), self.style.WHITE2)

    if self._terminal_width < 60:
        # Compact format for small screens
        self.print(f"\n[{color_func(state)}]")
        result = f"\n[{state}]"
    else:
        # Full format for larger screens
        self.print(f"\n{self.style.Bold('State:')} {color_func(state)}")
        result = f"\nState: {state}"

    if details:
        for key, value in details.items():
            # Truncate long values on small screens
            if self._terminal_width < 60 and len(str(value)) > 30:
                display_value = str(value)[:27] + "..."
            else:
                display_value = str(value)

            if self._terminal_width < 60:
                self.print(f"  {key}: {display_value}")
                result += f"\n  {key}: {display_value}"
            else:
                self.print(f"  {self.style.GREY('├─')} {self.style.CYAN(key)}: {display_value}")
                result += f"\n  ├─ {key}: {display_value}"

    return result

print_success(message)

Print success message with consistent formatting

Source code in toolboxv2/mods/isaa/extras/verbose_output.py

def print_success(self, message: str):
    """Print success message with consistent formatting"""
    self.print(f"{self.style.GREEN('✓')} {self.style.GREEN(message)}")

print_table(headers, rows)

Print a dynamic table that adapts to screen size

Source code in toolboxv2/mods/isaa/extras/verbose_output.py

def print_table(self, headers: list[str], rows: list[list[str]]):
    """Print a dynamic table that adapts to screen size"""
    self._terminal_width = self._get_terminal_width()

    if not rows:
        return

    # Calculate column widths
    all_data = [headers] + rows
    col_widths = []

    for col in range(len(headers)):
        max_width = max(len(str(row[col])) for row in all_data if col < len(row))
        col_widths.append(min(max_width, self._terminal_width // len(headers) - 2))

    # Adjust if total width exceeds terminal
    total_width = sum(col_widths) + len(headers) * 3 + 1
    if total_width > self._terminal_width:
        # Proportionally reduce column widths
        scale_factor = (self._terminal_width - len(headers) * 3 - 1) / sum(col_widths)
        col_widths = [max(8, int(w * scale_factor)) for w in col_widths]

    # Print table
    self._print_table_row(headers, col_widths, is_header=True)
    self._print_table_separator(col_widths)

    for row in rows:
        self._print_table_row(row, col_widths)

print_warning(message)

Print warning message with consistent formatting

Source code in toolboxv2/mods/isaa/extras/verbose_output.py

def print_warning(self, message: str):
    """Print warning message with consistent formatting"""
    self.print(f"{self.style.YELLOW('⚠')} {self.style.YELLOW(message)}")

process_with_spinner(message, coroutine) async

Execute coroutine with adaptive spinner

Source code in toolboxv2/mods/isaa/extras/verbose_output.py

async def process_with_spinner(self, message: str, coroutine):
    """Execute coroutine with adaptive spinner"""
    self._terminal_width = self._get_terminal_width()

    if self._terminal_width < 60:
        # Simple spinner for small screens
        spinner_symbols = "⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏"
    else:
        # Detailed spinner for larger screens
        spinner_symbols = "⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏"

    # Truncate message if too long
    if len(message) > self._terminal_width - 10:
        display_message = message[:self._terminal_width - 13] + "..."
    else:
        display_message = message

    with Spinner(f"{self.style.CYAN('●')} {display_message}", symbols=spinner_symbols):
        return await coroutine

EnhancedVerboseOutput

Main interface for verbose output with full functionality

Source code in toolboxv2/mods/isaa/extras/verbose_output.py

class EnhancedVerboseOutput:
    """Main interface for verbose output with full functionality"""

    def __init__(self, verbose: bool = True, print_func=None, **formatter_kwargs):
        self.verbose = verbose
        self.print = print_func or print
        self.formatter = DynamicVerboseFormatter(self.print, **formatter_kwargs)
        self._start_time = time.time()

    def __getattr__(self, name):
        """Delegate to formatter for convenience"""
        return getattr(self.formatter, name)

    async def print_agent_response(self, response: str):
        await self.log_message("assistant", response)

    async def print_thought(self, thought: str):
        await self.log_message("assistant", f"Thought: {thought}")

    async def log_message(self, role: str, content: str):
        """Log chat messages with role-based formatting"""
        if not self.verbose:
            return

        role_formats = {
            'user': (self.formatter.style.GREEN, "👤"),
            'assistant': (self.formatter.style.BLUE, "🤖"),
            'system': (self.formatter.style.YELLOW, "⚙️"),
            'error': (self.formatter.style.RED, "❌"),
            'debug': (self.formatter.style.GREY, "🐛")
        }

        color_func, icon = role_formats.get(role.lower(), (self.formatter.style.WHITE, "•"))

        if content.startswith("```"):
            self.formatter.print_code_block(content)
            return

        if content.startswith("{") or content.startswith("[") and content.endswith("}") or content.endswith("]"):
            content = json.dumps(json.loads(content), indent=2)

        # Adapt formatting based on screen size
        if self.formatter._terminal_width < 60:
            self.print(f"\n{icon} [{role.upper()}]")
            # Wrap content for small screens
            wrapped_content = self.formatter._wrap_text(content, self.formatter._terminal_width - 2)
            for line in wrapped_content:
                self.print(f"  {line}")
        else:
            self.print(f"\n{icon} {color_func(f'[{role.upper()}]')}")
            self.print(f"{self.formatter.style.GREY('└─')} {content}")
        self.print()

    async def log_process_result(self, result: dict[str, Any]):
        """Log processing results with structured formatting"""
        if not self.verbose:
            return

        content_parts = []

        if 'action' in result:
            content_parts.append(f"Action: {result['action']}")
        if 'is_completed' in result:
            content_parts.append(f"Completed: {result['is_completed']}")
        if 'effectiveness' in result:
            content_parts.append(f"Effectiveness: {result['effectiveness']}")
        if 'recommendations' in result:
            content_parts.append(f"Recommendations:\n{result['recommendations']}")
        if 'workflow' in result:
            content_parts.append(f"Workflow:\n{result['workflow']}")
        if 'errors' in result and result['errors']:
            content_parts.append(f"Errors: {result['errors']}")
        if 'content' in result:
            content_parts.append(f"Content:\n{result['content']}")

        self.formatter.print_section("Process Result", '\n'.join(content_parts))

    def log_header(self, text: str):
        """Log header with timing information"""
        if not self.verbose:
            return

        elapsed = time.time() - self._start_time
        timing = f" ({elapsed / 60:.1f}m)" if elapsed > 60 else f" ({elapsed:.1f}s)"

        self.formatter.print_header(f"{text}{timing}")

    def log_state(self, state: str, user_ns: dict = None, override: bool = False):
        """Log state with optional override"""
        if not self.verbose and not override:
            return

        return self.formatter.print_state(state, user_ns)

    async def process(self, message: str, coroutine):
        """Process with optional spinner"""
        if not self.verbose:
            return await coroutine

        if message.lower() in ["code", "silent"]:
            return await coroutine

        return await self.formatter.process_with_spinner(message, coroutine)

    def print_tool_call(self, tool_name: str, tool_args: dict, result: str | None = None):
        """
        Gibt Informationen zum Tool-Aufruf aus.
        Versucht, das Ergebnis als JSON zu formatieren, wenn möglich.
        """
        if not self.verbose:
            return

        # Argumente wie zuvor formatieren
        args_str = json.dumps(tool_args, indent=2, ensure_ascii=False) if tool_args else "None"
        content = f"Tool: {tool_name}\nArguments:\n{args_str}"

        if result:
            result_output = ""
            try:
                # 1. Versuch, den String als JSON zu parsen
                data = json.loads(result)

                # 2. Prüfen, ob das Ergebnis ein Dictionary ist (der häufigste Fall)
                if isinstance(data, dict):
                    # Eine Kopie für die Anzeige erstellen, um den 'output'-Wert zu ersetzen
                    display_data = data.copy()
                    output_preview = ""

                    # Spezielle Handhabung für einen langen 'output'-String, falls vorhanden
                    if 'output' in display_data and isinstance(display_data['output'], str):
                        full_output = display_data['output']
                        # Den langen String im JSON durch einen Platzhalter ersetzen
                        display_data['output'] = "<-- [Inhalt wird separat formatiert]"

                        # Vorschau mit den ersten 3 Zeilen erstellen
                        lines = full_output.strip().split('\n')[:3]
                        preview_text = '\n'.join(lines)
                        output_preview = f"\n\n--- Vorschau für 'output' ---\n\x1b[90m{preview_text}\n...\x1b[0m"  # Hellgrauer Text
                        # display_data['output'] = output_preview
                    # Das formatierte JSON (mit Platzhalter) zum Inhalt hinzufügen
                    formatted_json = json.dumps(display_data, indent=2, ensure_ascii=False)
                    result_output = f"Geparstes Dictionary:\n{formatted_json}{output_preview}"

                else:
                    # Falls es valides JSON, aber kein Dictionary ist (z.B. eine Liste)
                    result_output = f"Gepastes JSON (kein Dictionary):\n{json.dumps(data, indent=2, ensure_ascii=False)}"

            except json.JSONDecodeError:
                # 3. Wenn Parsen fehlschlägt, den String als Rohtext behandeln
                result_output = f"{result}"

            content += f"\nResult:\n{result_output}"

        else:
            # Fall, wenn der Task noch läuft
            content += "\nResult: In progress..."

        # Den gesamten Inhalt an den Formatter übergeben
        self.formatter.print_section("Tool Call", content)

    def print_event(self, event: dict):
        """Print event information"""
        if not self.verbose:
            return

        if event.get("content") and event["content"].get("parts"):
            for part in event["content"]["parts"]:
                if part.get("text"):
                    self.formatter.print_info(f"Thought: {part['text']}")
                if part.get("function_call"):
                    self.print_tool_call(
                        part["function_call"]["name"],
                        part["function_call"]["args"]
                    )
                if part.get("function_response"):
                    result = part["function_response"]["response"].get("result", "")
                    self.print_tool_call(
                        part["function_response"]["name"],
                        {},
                        str(result)
                    )

        if event.get("usage_metadata"):
            self.formatter.print_info(f"Token usage: {event['usage_metadata']}")

    @contextmanager
    def section_context(self, title: str):
        """Context manager for sections"""
        if self.verbose:
            self.formatter.print_section(title, "Starting...")
        try:
            yield
        finally:
            if self.verbose:
                self.formatter.print_success(f"Completed: {title}")

    def clear_line(self):
        """Clear current line"""
        self.print('\r' + ' ' * self.formatter._terminal_width + '\r', end='')

    def print_separator(self, char: str = "─"):
        """Print a separator line"""
        self.print(self.formatter.style.GREY(char * self.formatter._terminal_width))

    def print_warning(self, message: str):
        """Print a warning message with yellow style"""
        if self.verbose:
            self.print(self.formatter.style.YELLOW(f"⚠️  WARNING: {message}"))

    def print_error(self, message: str):
        """Print an error message with red style"""
        if self.verbose:
            self.print(self.formatter.style.RED(f"❌ ERROR: {message}"))

    def print_success(self, message: str):
        """Print a success message with green style"""
        if self.verbose:
            self.print(self.formatter.style.GREEN(f"✅ SUCCESS: {message}"))

__getattr__(name)

Delegate to formatter for convenience

Source code in toolboxv2/mods/isaa/extras/verbose_output.py

def __getattr__(self, name):
    """Delegate to formatter for convenience"""
    return getattr(self.formatter, name)

clear_line()

Clear current line

Source code in toolboxv2/mods/isaa/extras/verbose_output.py

def clear_line(self):
    """Clear current line"""
    self.print('\r' + ' ' * self.formatter._terminal_width + '\r', end='')

log_header(text)

Log header with timing information

Source code in toolboxv2/mods/isaa/extras/verbose_output.py

def log_header(self, text: str):
    """Log header with timing information"""
    if not self.verbose:
        return

    elapsed = time.time() - self._start_time
    timing = f" ({elapsed / 60:.1f}m)" if elapsed > 60 else f" ({elapsed:.1f}s)"

    self.formatter.print_header(f"{text}{timing}")

log_message(role, content) async

Log chat messages with role-based formatting

Source code in toolboxv2/mods/isaa/extras/verbose_output.py

async def log_message(self, role: str, content: str):
    """Log chat messages with role-based formatting"""
    if not self.verbose:
        return

    role_formats = {
        'user': (self.formatter.style.GREEN, "👤"),
        'assistant': (self.formatter.style.BLUE, "🤖"),
        'system': (self.formatter.style.YELLOW, "⚙️"),
        'error': (self.formatter.style.RED, "❌"),
        'debug': (self.formatter.style.GREY, "🐛")
    }

    color_func, icon = role_formats.get(role.lower(), (self.formatter.style.WHITE, "•"))

    if content.startswith("```"):
        self.formatter.print_code_block(content)
        return

    if content.startswith("{") or content.startswith("[") and content.endswith("}") or content.endswith("]"):
        content = json.dumps(json.loads(content), indent=2)

    # Adapt formatting based on screen size
    if self.formatter._terminal_width < 60:
        self.print(f"\n{icon} [{role.upper()}]")
        # Wrap content for small screens
        wrapped_content = self.formatter._wrap_text(content, self.formatter._terminal_width - 2)
        for line in wrapped_content:
            self.print(f"  {line}")
    else:
        self.print(f"\n{icon} {color_func(f'[{role.upper()}]')}")
        self.print(f"{self.formatter.style.GREY('└─')} {content}")
    self.print()

log_process_result(result) async

Log processing results with structured formatting

Source code in toolboxv2/mods/isaa/extras/verbose_output.py

async def log_process_result(self, result: dict[str, Any]):
    """Log processing results with structured formatting"""
    if not self.verbose:
        return

    content_parts = []

    if 'action' in result:
        content_parts.append(f"Action: {result['action']}")
    if 'is_completed' in result:
        content_parts.append(f"Completed: {result['is_completed']}")
    if 'effectiveness' in result:
        content_parts.append(f"Effectiveness: {result['effectiveness']}")
    if 'recommendations' in result:
        content_parts.append(f"Recommendations:\n{result['recommendations']}")
    if 'workflow' in result:
        content_parts.append(f"Workflow:\n{result['workflow']}")
    if 'errors' in result and result['errors']:
        content_parts.append(f"Errors: {result['errors']}")
    if 'content' in result:
        content_parts.append(f"Content:\n{result['content']}")

    self.formatter.print_section("Process Result", '\n'.join(content_parts))

log_state(state, user_ns=None, override=False)

Log state with optional override

Source code in toolboxv2/mods/isaa/extras/verbose_output.py

def log_state(self, state: str, user_ns: dict = None, override: bool = False):
    """Log state with optional override"""
    if not self.verbose and not override:
        return

    return self.formatter.print_state(state, user_ns)

print_error(message)

Print an error message with red style

Source code in toolboxv2/mods/isaa/extras/verbose_output.py

def print_error(self, message: str):
    """Print an error message with red style"""
    if self.verbose:
        self.print(self.formatter.style.RED(f"❌ ERROR: {message}"))

print_event(event)

Print event information

Source code in toolboxv2/mods/isaa/extras/verbose_output.py

def print_event(self, event: dict):
    """Print event information"""
    if not self.verbose:
        return

    if event.get("content") and event["content"].get("parts"):
        for part in event["content"]["parts"]:
            if part.get("text"):
                self.formatter.print_info(f"Thought: {part['text']}")
            if part.get("function_call"):
                self.print_tool_call(
                    part["function_call"]["name"],
                    part["function_call"]["args"]
                )
            if part.get("function_response"):
                result = part["function_response"]["response"].get("result", "")
                self.print_tool_call(
                    part["function_response"]["name"],
                    {},
                    str(result)
                )

    if event.get("usage_metadata"):
        self.formatter.print_info(f"Token usage: {event['usage_metadata']}")

print_separator(char='─')

Print a separator line

Source code in toolboxv2/mods/isaa/extras/verbose_output.py

def print_separator(self, char: str = "─"):
    """Print a separator line"""
    self.print(self.formatter.style.GREY(char * self.formatter._terminal_width))

print_success(message)

Print a success message with green style

Source code in toolboxv2/mods/isaa/extras/verbose_output.py

def print_success(self, message: str):
    """Print a success message with green style"""
    if self.verbose:
        self.print(self.formatter.style.GREEN(f"✅ SUCCESS: {message}"))

print_tool_call(tool_name, tool_args, result=None)

Gibt Informationen zum Tool-Aufruf aus. Versucht, das Ergebnis als JSON zu formatieren, wenn möglich.

Source code in toolboxv2/mods/isaa/extras/verbose_output.py

def print_tool_call(self, tool_name: str, tool_args: dict, result: str | None = None):
    """
    Gibt Informationen zum Tool-Aufruf aus.
    Versucht, das Ergebnis als JSON zu formatieren, wenn möglich.
    """
    if not self.verbose:
        return

    # Argumente wie zuvor formatieren
    args_str = json.dumps(tool_args, indent=2, ensure_ascii=False) if tool_args else "None"
    content = f"Tool: {tool_name}\nArguments:\n{args_str}"

    if result:
        result_output = ""
        try:
            # 1. Versuch, den String als JSON zu parsen
            data = json.loads(result)

            # 2. Prüfen, ob das Ergebnis ein Dictionary ist (der häufigste Fall)
            if isinstance(data, dict):
                # Eine Kopie für die Anzeige erstellen, um den 'output'-Wert zu ersetzen
                display_data = data.copy()
                output_preview = ""

                # Spezielle Handhabung für einen langen 'output'-String, falls vorhanden
                if 'output' in display_data and isinstance(display_data['output'], str):
                    full_output = display_data['output']
                    # Den langen String im JSON durch einen Platzhalter ersetzen
                    display_data['output'] = "<-- [Inhalt wird separat formatiert]"

                    # Vorschau mit den ersten 3 Zeilen erstellen
                    lines = full_output.strip().split('\n')[:3]
                    preview_text = '\n'.join(lines)
                    output_preview = f"\n\n--- Vorschau für 'output' ---\n\x1b[90m{preview_text}\n...\x1b[0m"  # Hellgrauer Text
                    # display_data['output'] = output_preview
                # Das formatierte JSON (mit Platzhalter) zum Inhalt hinzufügen
                formatted_json = json.dumps(display_data, indent=2, ensure_ascii=False)
                result_output = f"Geparstes Dictionary:\n{formatted_json}{output_preview}"

            else:
                # Falls es valides JSON, aber kein Dictionary ist (z.B. eine Liste)
                result_output = f"Gepastes JSON (kein Dictionary):\n{json.dumps(data, indent=2, ensure_ascii=False)}"

        except json.JSONDecodeError:
            # 3. Wenn Parsen fehlschlägt, den String als Rohtext behandeln
            result_output = f"{result}"

        content += f"\nResult:\n{result_output}"

    else:
        # Fall, wenn der Task noch läuft
        content += "\nResult: In progress..."

    # Den gesamten Inhalt an den Formatter übergeben
    self.formatter.print_section("Tool Call", content)

print_warning(message)

Print a warning message with yellow style

Source code in toolboxv2/mods/isaa/extras/verbose_output.py

def print_warning(self, message: str):
    """Print a warning message with yellow style"""
    if self.verbose:
        self.print(self.formatter.style.YELLOW(f"⚠️  WARNING: {message}"))

process(message, coroutine) async

Process with optional spinner

Source code in toolboxv2/mods/isaa/extras/verbose_output.py

async def process(self, message: str, coroutine):
    """Process with optional spinner"""
    if not self.verbose:
        return await coroutine

    if message.lower() in ["code", "silent"]:
        return await coroutine

    return await self.formatter.process_with_spinner(message, coroutine)

section_context(title)

Context manager for sections

Source code in toolboxv2/mods/isaa/extras/verbose_output.py

@contextmanager
def section_context(self, title: str):
    """Context manager for sections"""
    if self.verbose:
        self.formatter.print_section(title, "Starting...")
    try:
        yield
    finally:
        if self.verbose:
            self.formatter.print_success(f"Completed: {title}")

web_search

clean_markdown_robust(content)

Robust markdown cleaning

Source code in toolboxv2/mods/isaa/extras/web_search.py

def clean_markdown_robust(content: str) -> str:
    """Robust markdown cleaning"""
    if not content:
        return ""

    # Remove common encoding artifacts more aggressively
    replacements = {
        '�': '',
        '’': "'", '“': '"', 'â€': '"', '…': '...',
        'â€"': '-', 'â€"': '--', 'Â': ' ',
        'á': 'á', 'é': 'é', 'í': 'í', 'ó': 'ó', 'ú': 'ú',
        '•': '•', '·': '·', '«': '«', '»': '»'
    }

    for old, new in replacements.items():
        content = content.replace(old, new)

    # Remove lines with too many non-ASCII characters
    lines = content.split('\n')
    cleaned_lines = []

    for line in lines:
        line = line.strip()
        if not line:
            cleaned_lines.append('')
            continue

        # Skip lines that are mostly garbled
        ascii_chars = sum(1 for c in line if ord(c) < 128)
        if len(line) > 10 and ascii_chars / len(line) < 0.7:
            continue

        # Skip navigation/junk lines
        if (len(line) < 3 or
            line.lower() in ['home', 'menu', 'search', 'login', 'register'] or
            re.match(r'^[\W\s]*$', line)):
            continue

        cleaned_lines.append(line)

    # Remove excessive empty lines
    result = '\n'.join(cleaned_lines)
    result = re.sub(r'\n{3,}', '\n\n', result)

    return result.strip()

convert_to_markdown(element)

Convert HTML element to markdown with fallbacks

Source code in toolboxv2/mods/isaa/extras/web_search.py

def convert_to_markdown(element):
    """Convert HTML element to markdown with fallbacks"""

    # Strategy 1: Use html2text
    try:
        import html2text
        h = html2text.HTML2Text()
        h.ignore_links = False
        h.ignore_images = True
        h.body_width = 0
        h.unicode_snob = True
        h.skip_internal_links = True
        h.inline_links = False
        h.decode_errors = 'ignore'

        markdown = h.handle(str(element))
        if markdown and len(markdown.strip()) > 100:
            return markdown
    except ImportError:
        print("html2text not installed")
    except:
        pass

    # Strategy 2: Extract text with basic formatting
    try:
        text_parts = []

        for elem in element.find_all(['h1', 'h2', 'h3', 'h4', 'h5', 'h6']):
            level = int(elem.name[1])
            text_parts.append('#' * level + ' ' + elem.get_text(strip=True))
            elem.replace_with('[HEADING_PLACEHOLDER]')

        for elem in element.find_all('p'):
            text = elem.get_text(strip=True)
            if text:
                text_parts.append(text)
            elem.replace_with('[PARAGRAPH_PLACEHOLDER]')

        # Get remaining text
        remaining_text = element.get_text(separator='\n', strip=True)

        # Combine all text
        all_text = '\n\n'.join(text_parts)
        if remaining_text:
            all_text += '\n\n' + remaining_text

        return all_text

    except:
        pass

    # Strategy 3: Simple text extraction
    return element.get_text(separator='\n', strip=True)

find_main_content(soup)

Find main content using multiple strategies

Source code in toolboxv2/mods/isaa/extras/web_search.py

def find_main_content(soup):
    """Find main content using multiple strategies"""

    # Strategy 1: Look for semantic HTML5 elements
    for tag in ['main', 'article']:
        element = soup.find(tag)
        if element and len(element.get_text(strip=True)) > 300:
            return element

    # Strategy 2: Look for common content containers
    content_selectors = [
        '[role="main"]', '.main-content', '#main-content', '.content', '#content',
        '.post-content', '.entry-content', '.article-content', '.blog-content',
        '.story-body', '.article-body', '.post-body'
    ]

    for selector in content_selectors:
        element = soup.select_one(selector)
        if element and len(element.get_text(strip=True)) > 300:
            return element

    # Strategy 3: Find the div with most text content
    divs = soup.find_all('div')
    if divs:
        content_divs = [(div, len(div.get_text(strip=True))) for div in divs]
        content_divs = [(div, length) for div, length in content_divs if length > 300]

        if content_divs:
            content_divs.sort(key=lambda x: x[1], reverse=True)
            return content_divs[0][0]

    # Strategy 4: Use body as fallback
    return soup.find('body')

is_content_parseable(content)

Check if content is properly parsed and readable

Source code in toolboxv2/mods/isaa/extras/web_search.py

def is_content_parseable(content: str) -> bool:
    """
    Check if content is properly parsed and readable
    """
    if not content or len(content.strip()) < 50:
        return False

    # Check for too many non-ASCII characters that look like encoding errors
    total_chars = len(content)
    if total_chars == 0:
        return False

    # Count problematic characters
    problematic_chars = 0
    replacement_chars = content.count('�')

    # Check for sequences of garbled characters
    garbled_patterns = [
        r'[ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõöøùúûüýþÿ]{5,}',
        r'[’“â€�]{3,}',
        r'[\x80-\xff]{4,}',  # High-byte sequences
        r'[^\x00-\x7F\s]{10,}'  # Too many non-ASCII chars in sequence
    ]

    for pattern in garbled_patterns:
        matches = re.findall(pattern, content)
        problematic_chars += sum(len(match) for match in matches)

    # Calculate ratios
    replacement_ratio = replacement_chars / total_chars
    problematic_ratio = problematic_chars / total_chars

    # Check for readable English content
    english_words = re.findall(r'\b[a-zA-Z]{3,}\b', content)
    english_ratio = len(' '.join(english_words)) / total_chars if english_words else 0

    # Criteria for parseable content
    is_parseable = (
        replacement_ratio < 0.05 and  # Less than 5% replacement chars
        problematic_ratio < 0.15 and  # Less than 15% garbled chars
        english_ratio > 0.3 and  # At least 30% English words
        len(english_words) > 10  # At least 10 English words
    )

    if not is_parseable:
        print("Content failed parseability check:")
        print(f"  Replacement ratio: {replacement_ratio:.1%}")
        print(f"  Problematic ratio: {problematic_ratio:.1%}")
        print(f"  English ratio: {english_ratio:.1%}")
        print(f"  English words: {len(english_words)}")

    return is_parseable

is_mostly_readable(text)

Check if text is mostly readable ASCII/common unicode

Source code in toolboxv2/mods/isaa/extras/web_search.py

def is_mostly_readable(text: str) -> bool:
    """Check if text is mostly readable ASCII/common unicode"""
    if not text:
        return False

    readable_chars = sum(1 for c in text if c.isprintable() or c.isspace())
    return readable_chars / len(text) > 0.8

robust_search()

Test the robust search functionality

Source code in toolboxv2/mods/isaa/extras/web_search.py

def robust_search():
    """Test the robust search functionality"""
    query = "Python web scraping best practices"
    results = web_search(query, max_results=3)

    print(f"\n{'=' * 60}")
    print(f"FINAL RESULTS FOR: '{query}'")
    print(f"{'=' * 60}")

    for i, result in enumerate(results, 1):
        print(f"\n{i}. {result['title']}")
        print(f"URL: {result['url']}")
        print(f"Content length: {len(result['content'])} characters")
        print(f"First 300 chars: {result['content'][:300]}...")

        # Show parseability stats
        content = result['content']
        ascii_ratio = sum(1 for c in content if ord(c) < 128) / len(content)
        print(f"ASCII ratio: {ascii_ratio:.1%}")
        print("-" * 80)

url_to_markdown_robust(url)

Robust URL to markdown converter with multiple encoding strategies

Source code in toolboxv2/mods/isaa/extras/web_search.py

def url_to_markdown_robust(url: str) -> str | None:
    """
    Robust URL to markdown converter with multiple encoding strategies
    """
    try:
        headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
            'Accept-Language': 'en-US,en;q=0.9',
            'Accept-Charset': 'utf-8, iso-8859-1;q=0.5',
            'Connection': 'keep-alive'
        }

        response = requests.get(url, headers=headers, timeout=20, allow_redirects=True)
        response.raise_for_status()

        # Quick content type check
        content_type = response.headers.get('content-type', '').lower()
        if not any(ct in content_type for ct in ['text/html', 'text/plain', 'application/xhtml']):
            print(f"Skipping non-HTML content: {content_type}")
            return None

        # Get raw content
        raw_content = response.content

        # Strategy 1: Try response encoding first if it looks reliable
        decoded_content = None
        used_encoding = None

        response_encoding = response.encoding
        if response_encoding and response_encoding.lower() not in ['iso-8859-1', 'ascii']:
            try:
                decoded_content = response.text
                used_encoding = response_encoding
                # Quick test for encoding quality
                if '�' in decoded_content or not is_mostly_readable(decoded_content[:1000]):
                    decoded_content = None
            except:
                pass

        # Strategy 2: Detect encoding from content
        if not decoded_content:
            try:
                import chardet
                detected = chardet.detect(raw_content)
                if detected and detected.get('confidence', 0) > 0.8:
                    decoded_content = raw_content.decode(detected['encoding'])
                    used_encoding = detected['encoding']
                    if '�' in decoded_content or not is_mostly_readable(decoded_content[:1000]):
                        decoded_content = None
            except ImportError and ModuleNotFoundError:
                print("chardet not installed")
            except:
                pass

        # Strategy 3: Extract encoding from HTML meta tags
        if not decoded_content:
            try:
                # Try UTF-8 first to read meta tags
                temp_content = raw_content.decode('utf-8', errors='ignore')[:2048]
                charset_patterns = [
                    r'<meta[^>]+charset["\'\s]*=["\'\s]*([^"\'>\s]+)',
                    r'<meta[^>]+content[^>]+charset=([^"\'>\s;]+)',
                    r'<\?xml[^>]+encoding["\'\s]*=["\'\s]*([^"\'>\s]+)'
                ]

                for pattern in charset_patterns:
                    match = re.search(pattern, temp_content, re.I)
                    if match:
                        encoding = match.group(1).strip().lower()
                        try:
                            decoded_content = raw_content.decode(encoding)
                            used_encoding = encoding
                            if not ('�' in decoded_content or not is_mostly_readable(decoded_content[:1000])):
                                break
                        except:
                            pass
                        decoded_content = None
            except:
                pass

        # Strategy 4: Try common encodings
        if not decoded_content:
            common_encodings = ['utf-8', 'utf-8-sig', 'latin1', 'cp1252', 'iso-8859-1']
            for encoding in common_encodings:
                try:
                    test_content = raw_content.decode(encoding)
                    if is_mostly_readable(test_content[:1000]) and '�' not in test_content[:1000]:
                        decoded_content = test_content
                        used_encoding = encoding
                        break
                except:
                    continue

        # Strategy 5: Last resort with error handling
        if not decoded_content:
            decoded_content = raw_content.decode('utf-8', errors='replace')
            used_encoding = 'utf-8 (with errors)'

        print(f"Used encoding: {used_encoding}")

        # Parse with BeautifulSoup
        soup = BeautifulSoup(decoded_content, 'html.parser')

        # Remove all unwanted elements aggressively
        unwanted_tags = ['script', 'style', 'nav', 'header', 'footer', 'aside', 'iframe',
                         'form', 'button', 'input', 'noscript', 'meta', 'link', 'svg']
        for tag in unwanted_tags:
            for element in soup.find_all(tag):
                element.decompose()

        # Remove elements with unwanted classes/ids
        unwanted_patterns = [
            r'.*ad[s]?[-_].*', r'.*banner.*', r'.*popup.*', r'.*modal.*',
            r'.*cookie.*', r'.*newsletter.*', r'.*social.*', r'.*share.*',
            r'.*comment.*', r'.*sidebar.*', r'.*menu.*', r'.*navigation.*'
        ]

        for pattern in unwanted_patterns:
            for attr in ['class', 'id']:
                for element in soup.find_all(attrs={attr: re.compile(pattern, re.I)}):
                    element.decompose()

        # Find main content with multiple strategies
        main_content = find_main_content(soup)

        if not main_content:
            print("No main content found")
            return None

        # Convert to markdown using multiple strategies
        markdown_content = convert_to_markdown(main_content)

        if not markdown_content:
            print("Markdown conversion failed")
            return None

        # Clean and validate
        cleaned_content = clean_markdown_robust(markdown_content)

        # Final validation
        if not is_content_parseable(cleaned_content):
            print("Content failed parseability check")
            return None

        return cleaned_content

    except Exception as e:
        print(f"Error processing {url}: {e}")
        return None

web_search(query, max_results=5)

Main search function with robust fallbacks

Source code in toolboxv2/mods/isaa/extras/web_search.py

def web_search(query: str, max_results: int = 5) -> list[dict[str, str]]:
    """
    Main search function with robust fallbacks
    """
    # Try API searches first if available
    api_keys = {
        'serpapi': os.getenv('SERPAPI_API_KEY'),
        'bing': os.getenv('BING_API_KEY')
    }
    if isinstance(max_results, str):
        if max_results.startswith('"') and max_results.endswith('"') or max_results.startswith("'") and max_results.endswith("'"):
            max_results = max_results[1:-1]
        max_results = int(max_results.strip())
    if api_keys:
        for api_name, api_key in api_keys.items():
            if api_key:
                try:
                    print(f"Trying {api_name.upper()} API...")
                    if api_name == 'serpapi':
                        results = web_search_serpapi(query, max_results, api_key)
                    elif api_name == 'bing':
                        results = web_search_bing(query, max_results, api_key)
                    else:
                        continue

                    if results and len(results) >= max_results:
                        return results
                except Exception as e:
                    print(f"{api_name.upper()} API failed: {e}")

    # Use robust DuckDuckGo search
    return web_search_robust(query, max_results)

web_search_bing(query, max_results=5, api_key=None)

Web search using Bing Search API (free tier: 3,000 queries/month) Get your free API key at: https://azure.microsoft.com/en-us/services/cognitive-services/bing-web-search-api/

Source code in toolboxv2/mods/isaa/extras/web_search.py

def web_search_bing(query: str, max_results: int = 5, api_key: str = None) -> list[dict[str, str]]:
    """
    Web search using Bing Search API (free tier: 3,000 queries/month)
    Get your free API key at: https://azure.microsoft.com/en-us/services/cognitive-services/bing-web-search-api/
    """
    if not api_key:
        print("Please get a free API key from Azure Cognitive Services")
        return []

    try:
        url = "https://api.bing.microsoft.com/v7.0/search"
        headers = {
            "Ocp-Apim-Subscription-Key": api_key
        }
        params = {
            "q": query,
            "count": max_results,
            "textDecorations": False,
            "textFormat": "HTML"
        }

        response = requests.get(url, headers=headers, params=params)
        response.raise_for_status()
        data = response.json()

        results = []
        if "webPages" in data and "value" in data["webPages"]:
            for result in data["webPages"]["value"][:max_results]:
                url_link = result.get("url", "")
                title = result.get("name", "")

                print(f"Processing: {title}")
                markdown_content = url_to_markdown_robust(url_link)

                if markdown_content:
                    results.append({
                        'url': url_link,
                        'title': title,
                        'content': markdown_content
                    })

                # time.sleep(1)

        return results

    except Exception as e:
        print(f"Bing search error: {e}")
        return []

web_search_robust(query, max_results=5, max_attempts=15)

Robust search that keeps trying until it gets enough good results

Source code in toolboxv2/mods/isaa/extras/web_search.py

def web_search_robust(query: str, max_results: int = 5, max_attempts: int = 15) -> list[dict[str, str]]:
    """
    Robust search that keeps trying until it gets enough good results
    """
    if isinstance(max_results, str):
        if max_results.startswith('"') and max_results.endswith('"') or max_results.startswith("'") and max_results.endswith("'"):
            max_results = max_results[1:-1]
        max_results = int(max_results.strip())
    if isinstance(max_attempts, str):
        if max_attempts.startswith('"') and max_attempts.endswith('"') or max_attempts.startswith("'") and max_attempts.endswith("'"):
            max_attempts = max_attempts[1:-1]
        max_attempts = int(max_attempts.strip())

    def get_more_search_urls(search_query: str, num_urls: int = 15) -> list[dict[str, str]]:
        """Get more URLs than needed so we can filter out bad ones"""
        try:
            headers = {
                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
                'Accept': 'text/html,application/xhtml+xml',
                'Accept-Language': 'en-US,en;q=0.9',
            }

            # Try DuckDuckGo lite
            search_url = "https://lite.duckduckgo.com/lite/"
            data = {'q': search_query}

            response = requests.post(search_url, data=data, headers=headers, timeout=15)
            response.raise_for_status()

            soup = BeautifulSoup(response.content, 'html.parser')
            results = []

            for link in soup.find_all('a', href=True):
                href = link.get('href', '')
                text = link.get_text(strip=True)

                if (href.startswith('http') and
                    'duckduckgo.com' not in href and
                    len(text) > 5 and
                    not any(skip in href.lower() for skip in ['ads', 'shopping', 'images'])):

                    results.append({
                        'url': href,
                        'title': text[:150]
                    })

                    if len(results) >= num_urls:
                        break

            return results

        except Exception as e:
            print(f"Search error: {e}")
            return []

    def get_fallback_urls(search_query: str) -> list[dict[str, str]]:
        """Get fallback URLs from known good sites"""
        encoded_query = quote_plus(search_query)
        fallback_urls = [
            f"https://stackoverflow.com/search?q={encoded_query}",
            f"https://www.reddit.com/search/?q={encoded_query}",
            f"https://medium.com/search?q={encoded_query}",
            f"https://dev.to/search?q={encoded_query}",
            f"https://github.com/search?q={encoded_query}&type=repositories",
            f"https://docs.python.org/3/search.html?q={encoded_query}",
            f"https://realpython.com/?s={encoded_query}",
            f"https://towardsdatascience.com/search?q={encoded_query}",
            f"https://www.geeksforgeeks.org/?s={encoded_query}",
            f"https://hackernoon.com/search?query={encoded_query}"
        ]

        return [
            {'url': url, 'title': f"Search results for '{search_query}'"}
            for url in fallback_urls
        ]

    print(f"Searching for: '{query}' (need {max_results} good results)")

    # Get candidate URLs
    candidate_urls = get_more_search_urls(query, max_attempts)

    if not candidate_urls:
        print("Primary search failed, using fallback URLs...")
        candidate_urls = get_fallback_urls(query)

    print(f"Found {len(candidate_urls)} candidate URLs")

    # Process URLs until we have enough good results
    good_results = []
    processed_count = 0

    def task(candidate):
        markdown_content = url_to_markdown_robust(candidate['url'])
        if markdown_content:
            return {
                'url': candidate['url'],
                'title': candidate['title'],
                'content': markdown_content
            }

    # runn all tasks in parallel
    with concurrent.futures.ThreadPoolExecutor() as executor:
        results = list(executor.map(task, candidate_urls))
        processed_count = len(candidate_urls)

    good_results = [result for result in results if result]

    #for candidate in candidate_urls:
    #    if len(good_results) >= max_results:
    #        break

    #    processed_count += 1
    #    print(f"\n[{processed_count}/{len(candidate_urls)}] Processing: {candidate['title'][:80]}...")

    #    markdown_content = url_to_markdown_robust(candidate['url'])

    #    if markdown_content:
    #        good_results.append({
    #            'url': candidate['url'],
    #            'title': candidate['title'],
    #            'content': markdown_content
    #        })
    #        print(f"✅ Success! Got result {len(good_results)}/{max_results}")
    #    else:
    #        print("❌ Skipped (unparseable or low quality)")

    #    # Small delay to be respectful
    #    time.sleep(1.5)

    print(f"\n🎉 Final results: {len(good_results)} good results out of {processed_count} attempted")
    return good_results

web_search_serpapi(query, max_results=5, api_key=None)

Web search using SerpAPI (free tier: 100 searches/month) Get your free API key at: https://serpapi.com/

Source code in toolboxv2/mods/isaa/extras/web_search.py

def web_search_serpapi(query: str, max_results: int = 5, api_key: str = None) -> list[dict[str, str]]:
    """
    Web search using SerpAPI (free tier: 100 searches/month)
    Get your free API key at: https://serpapi.com/
    """
    if not api_key:
        print("Please get a free API key from https://serpapi.com/")
        return []

    try:
        url = "https://serpapi.com/search"
        params = {
            "engine": "google",
            "q": query,
            "api_key": api_key,
            "num": max_results
        }

        response = requests.get(url, params=params)
        response.raise_for_status()
        data = response.json()

        results = []
        if "organic_results" in data:
            for result in data["organic_results"][:max_results]:
                url_link = result.get("link", "")
                title = result.get("title", "")

                print(f"Processing: {title}")
                markdown_content = url_to_markdown_robust(url_link)

                if markdown_content:
                    results.append({
                        'url': url_link,
                        'title': title,
                        'content': markdown_content
                    })

                #time.sleep(1)  # Be respectful

        return results

    except Exception as e:
        print(f"SerpAPI search error: {e}")
        return []

module

EnhancedAgentRequestHandler

Bases: BaseHTTPRequestHandler

Enhanced HTTP request handler for standalone server with comprehensive UI support.

Source code in toolboxv2/mods/isaa/module.py

class EnhancedAgentRequestHandler(BaseHTTPRequestHandler):
    """Enhanced HTTP request handler for standalone server with comprehensive UI support."""

    def __init__(self, isaa_mod, agent_id: str, agent, *args, **kwargs):
        self.isaa_mod = isaa_mod
        self.agent_id = agent_id
        self.agent = agent
        super().__init__(*args, **kwargs)

    def do_GET(self):
        """Handle GET requests for enhanced UI and status."""
        parsed_path = urlparse(self.path)

        if parsed_path.path in ['/', '/ui']:
            self._serve_enhanced_ui()
        elif parsed_path.path in ['/api/status', '/api/agent_ui/status', '/status']:
            self._serve_status()
        else:
            self._send_404()

    def do_POST(self):
        """Handle POST requests for enhanced API endpoints."""
        parsed_path = urlparse(self.path)

        if parsed_path.path in ['/api/run', '/api/agent_ui/run_agent']:
            self._handle_run_request()
        elif parsed_path.path in ['/api/reset', '/api/agent_ui/reset_context']:
            self._handle_reset_request()
        else:
            self._send_404()

    def _serve_enhanced_ui(self):
        """Serve the enhanced UI HTML."""
        try:
            html_content = get_agent_ui_html()

            self.send_response(200)
            self.send_header('Content-type', 'text/html')
            self.send_header('Content-Length', str(len(html_content.encode('utf-8'))))
            self.end_headers()
            self.wfile.write(html_content.encode('utf-8'))

        except Exception as e:
            self._send_error_response(500, f"Error serving UI: {str(e)}")

    def _serve_status(self):
        """Serve enhanced status information."""
        try:
            status_info = {
                'agent_id': self.agent_id,
                'agent_name': getattr(self.agent, 'name', 'Unknown'),
                'agent_type': self.agent.__class__.__name__,
                'status': 'active',
                'server_type': 'standalone',
                'timestamp': time.time()
            }

            if hasattr(self.agent, 'status'):
                try:
                    agent_status = self.agent.status()
                    if isinstance(agent_status, dict):
                        status_info['agent_status'] = agent_status
                except:
                    pass

            response_data = json.dumps(status_info).encode('utf-8')

            self.send_response(200)
            self.send_header('Content-type', 'application/json')
            self.send_header('Content-Length', str(len(response_data)))
            self.end_headers()
            self.wfile.write(response_data)

        except Exception as e:
            self._send_error_response(500, f"Error getting status: {str(e)}")

    def _handle_run_request(self):
        """Handle enhanced run requests with comprehensive progress tracking."""
        try:
            content_length = int(self.headers['Content-Length'])
            request_body = self.rfile.read(content_length)
            request_data = json.loads(request_body.decode('utf-8'))

            query = request_data.get('query', '')
            session_id = request_data.get('session_id', f'standalone_{secrets.token_hex(8)}')
            include_progress = request_data.get('include_progress', False)

            if not query:
                self._send_error_response(400, "Missing 'query' field")
                return

            # Run agent with enhanced progress tracking
            loop = asyncio.new_event_loop()
            asyncio.set_event_loop(loop)

            try:
                progress_tracker = EnhancedProgressTracker()
                progress_events = []
                enhanced_progress = {}

                async def standalone_progress_callback(event: ProgressEvent):
                    if include_progress:
                        progress_data = progress_tracker.extract_progress_data(event)
                        progress_events.append({
                            'timestamp': event.timestamp,
                            'event_type': event.event_type,
                            'status': getattr(event, 'status', 'unknown').value if hasattr(event, 'status') and event.status else 'unknown',
                            'data': event.to_dict()
                        })
                        enhanced_progress.update(progress_data)

                # Set progress callback
                original_callback = getattr(self.agent, 'progress_callback', None)

                if hasattr(self.agent, 'set_progress_callback'):
                    self.agent.set_progress_callback(standalone_progress_callback)
                elif hasattr(self.agent, 'progress_callback'):
                    self.agent.progress_callback = standalone_progress_callback

                # Execute agent
                result = loop.run_until_complete(
                    self.agent.a_run(query=query, session_id=session_id)
                )

                # Restore callback
                if hasattr(self.agent, 'set_progress_callback'):
                    self.agent.set_progress_callback(original_callback)
                elif hasattr(self.agent, 'progress_callback'):
                    self.agent.progress_callback = original_callback

                # Create enhanced response
                response_data = {
                    'success': True,
                    'result': result,
                    'session_id': session_id,
                    'agent_id': self.agent_id,
                    'server_type': 'standalone',
                    'timestamp': time.time()
                }

                if include_progress:
                    response_data.update({
                        'progress_events': progress_events,
                        'enhanced_progress': enhanced_progress,
                        'final_summary': progress_tracker.get_final_summary()
                    })
                self._send_json_response(response_data)

            finally:
                loop.close()

        except Exception as e:
            self._send_error_response(500, f"Execution error: {str(e)}")
            import traceback
            print(traceback.format_exc())

    def _handle_reset_request(self):
        """Handle enhanced reset requests."""
        try:
            success = False
            message = "Reset not supported"

            if hasattr(self.agent, 'clear_context'):
                self.agent.clear_context()
                success = True
                message = "Context reset successfully"
            elif hasattr(self.agent, 'reset'):
                self.agent.reset()
                success = True
                message = "Agent reset successfully"

            response_data = {
                'success': success,
                'message': message,
                'agent_id': self.agent_id,
                'timestamp': time.time()
            }

            self._send_json_response(response_data)

        except Exception as e:
            self._send_error_response(500, f"Reset error: {str(e)}")

    def _send_json_response(self, data: dict):
        """Send JSON response with CORS headers."""
        response_body = json.dumps(data, cls=CustomJSONEncoder).encode('utf-8')

        self.send_response(200)
        self.send_header('Content-type', 'application/json')
        self.send_header('Content-Length', str(len(response_body)))
        self.send_header('Access-Control-Allow-Origin', '*')
        self.send_header('Access-Control-Allow-Methods', 'GET, POST, OPTIONS')
        self.send_header('Access-Control-Allow-Headers', 'Content-Type')
        self.end_headers()
        self.wfile.write(response_body)

    def _send_error_response(self, code: int, message: str):
        """Send error response."""
        error_data = {'success': False, 'error': message, 'code': code}
        response_body = json.dumps(error_data).encode('utf-8')

        self.send_response(code)
        self.send_header('Content-type', 'application/json')
        self.send_header('Content-Length', str(len(response_body)))
        self.end_headers()
        self.wfile.write(response_body)

    def _send_404(self):
        """Send 404 response."""
        self._send_error_response(404, "Not Found")

    def log_message(self, format, *args):
        """Override to reduce logging noise."""
        pass

    def do_OPTIONS(self):
        """Handle preflight CORS requests."""
        self.send_response(200)
        self.send_header('Access-Control-Allow-Origin', '*')
        self.send_header('Access-Control-Allow-Methods', 'GET, POST, OPTIONS')
        self.send_header('Access-Control-Allow-Headers', 'Content-Type')
        self.end_headers()

do_GET()

Handle GET requests for enhanced UI and status.

Source code in toolboxv2/mods/isaa/module.py

def do_GET(self):
    """Handle GET requests for enhanced UI and status."""
    parsed_path = urlparse(self.path)

    if parsed_path.path in ['/', '/ui']:
        self._serve_enhanced_ui()
    elif parsed_path.path in ['/api/status', '/api/agent_ui/status', '/status']:
        self._serve_status()
    else:
        self._send_404()

do_OPTIONS()

Handle preflight CORS requests.

Source code in toolboxv2/mods/isaa/module.py

def do_OPTIONS(self):
    """Handle preflight CORS requests."""
    self.send_response(200)
    self.send_header('Access-Control-Allow-Origin', '*')
    self.send_header('Access-Control-Allow-Methods', 'GET, POST, OPTIONS')
    self.send_header('Access-Control-Allow-Headers', 'Content-Type')
    self.end_headers()

do_POST()

Handle POST requests for enhanced API endpoints.

Source code in toolboxv2/mods/isaa/module.py

def do_POST(self):
    """Handle POST requests for enhanced API endpoints."""
    parsed_path = urlparse(self.path)

    if parsed_path.path in ['/api/run', '/api/agent_ui/run_agent']:
        self._handle_run_request()
    elif parsed_path.path in ['/api/reset', '/api/agent_ui/reset_context']:
        self._handle_reset_request()
    else:
        self._send_404()

log_message(format, *args)

Override to reduce logging noise.

Source code in toolboxv2/mods/isaa/module.py

def log_message(self, format, *args):
    """Override to reduce logging noise."""
    pass

EnhancedProgressTracker

Enhanced progress tracker for detailed UI updates.

Source code in toolboxv2/mods/isaa/module.py

class EnhancedProgressTracker:
    """Enhanced progress tracker for detailed UI updates."""

    def __init__(self):
        self.session_state = {}
        self.last_outline_update = None
        self.last_activity_update = None

    def extract_progress_data(self, event: ProgressEvent) -> dict[str, Any]:
        """Extract comprehensive progress data from event."""
        progress_data = {}

        # Outline progress
        if hasattr(event, 'outline_data') or 'outline' in event.metadata:
            outline_info = getattr(event, 'outline_data', event.metadata.get('outline', {}))
            progress_data['outline'] = {
                'current_step': outline_info.get('current_step', 'Unknown'),
                'total_steps': outline_info.get('total_steps', 0),
                'step_name': outline_info.get('step_name', 'Processing'),
                'progress_percentage': outline_info.get('progress_percentage', 0),
                'substeps': outline_info.get('substeps', []),
                'estimated_completion': outline_info.get('estimated_completion')
            }

        # Activity information
        if hasattr(event, 'activity_data') or 'activity' in event.metadata:
            activity_info = getattr(event, 'activity_data', event.metadata.get('activity', {}))
            progress_data['activity'] = {
                'current_action': activity_info.get('current_action', 'Processing'),
                'action_details': activity_info.get('action_details', ''),
                'start_time': activity_info.get('start_time'),
                'elapsed_time': activity_info.get('elapsed_time'),
                'expected_duration': activity_info.get('expected_duration')
            }

        # Meta tool information
        if hasattr(event, 'meta_tool_data') or 'meta_tool' in event.metadata:
            meta_tool_info = getattr(event, 'meta_tool_data', event.metadata.get('meta_tool', {}))
            progress_data['meta_tool'] = {
                'tool_name': meta_tool_info.get('tool_name', 'Unknown'),
                'tool_status': meta_tool_info.get('tool_status', 'active'),
                'tool_input': meta_tool_info.get('tool_input', ''),
                'tool_output': meta_tool_info.get('tool_output', ''),
                'execution_time': meta_tool_info.get('execution_time')
            }

        # System status
        if hasattr(event, 'system_data') or 'system' in event.metadata:
            system_info = getattr(event, 'system_data', event.metadata.get('system', {}))
            progress_data['system'] = {
                'memory_usage': system_info.get('memory_usage', 0),
                'cpu_usage': system_info.get('cpu_usage', 0),
                'active_threads': system_info.get('active_threads', 1),
                'queue_size': system_info.get('queue_size', 0)
            }

        # Graph/workflow information
        if hasattr(event, 'graph_data') or 'graph' in event.metadata:
            graph_info = getattr(event, 'graph_data', event.metadata.get('graph', {}))
            progress_data['graph'] = {
                'current_node': graph_info.get('current_node', 'Unknown'),
                'completed_nodes': graph_info.get('completed_nodes', []),
                'remaining_nodes': graph_info.get('remaining_nodes', []),
                'node_connections': graph_info.get('node_connections', []),
                'execution_path': graph_info.get('execution_path', [])
            }

        return progress_data

extract_progress_data(event)

Extract comprehensive progress data from event.

Source code in toolboxv2/mods/isaa/module.py

def extract_progress_data(self, event: ProgressEvent) -> dict[str, Any]:
    """Extract comprehensive progress data from event."""
    progress_data = {}

    # Outline progress
    if hasattr(event, 'outline_data') or 'outline' in event.metadata:
        outline_info = getattr(event, 'outline_data', event.metadata.get('outline', {}))
        progress_data['outline'] = {
            'current_step': outline_info.get('current_step', 'Unknown'),
            'total_steps': outline_info.get('total_steps', 0),
            'step_name': outline_info.get('step_name', 'Processing'),
            'progress_percentage': outline_info.get('progress_percentage', 0),
            'substeps': outline_info.get('substeps', []),
            'estimated_completion': outline_info.get('estimated_completion')
        }

    # Activity information
    if hasattr(event, 'activity_data') or 'activity' in event.metadata:
        activity_info = getattr(event, 'activity_data', event.metadata.get('activity', {}))
        progress_data['activity'] = {
            'current_action': activity_info.get('current_action', 'Processing'),
            'action_details': activity_info.get('action_details', ''),
            'start_time': activity_info.get('start_time'),
            'elapsed_time': activity_info.get('elapsed_time'),
            'expected_duration': activity_info.get('expected_duration')
        }

    # Meta tool information
    if hasattr(event, 'meta_tool_data') or 'meta_tool' in event.metadata:
        meta_tool_info = getattr(event, 'meta_tool_data', event.metadata.get('meta_tool', {}))
        progress_data['meta_tool'] = {
            'tool_name': meta_tool_info.get('tool_name', 'Unknown'),
            'tool_status': meta_tool_info.get('tool_status', 'active'),
            'tool_input': meta_tool_info.get('tool_input', ''),
            'tool_output': meta_tool_info.get('tool_output', ''),
            'execution_time': meta_tool_info.get('execution_time')
        }

    # System status
    if hasattr(event, 'system_data') or 'system' in event.metadata:
        system_info = getattr(event, 'system_data', event.metadata.get('system', {}))
        progress_data['system'] = {
            'memory_usage': system_info.get('memory_usage', 0),
            'cpu_usage': system_info.get('cpu_usage', 0),
            'active_threads': system_info.get('active_threads', 1),
            'queue_size': system_info.get('queue_size', 0)
        }

    # Graph/workflow information
    if hasattr(event, 'graph_data') or 'graph' in event.metadata:
        graph_info = getattr(event, 'graph_data', event.metadata.get('graph', {}))
        progress_data['graph'] = {
            'current_node': graph_info.get('current_node', 'Unknown'),
            'completed_nodes': graph_info.get('completed_nodes', []),
            'remaining_nodes': graph_info.get('remaining_nodes', []),
            'node_connections': graph_info.get('node_connections', []),
            'execution_path': graph_info.get('execution_path', [])
        }

    return progress_data

Tools

Bases: MainTool, FileHandler

Source code in toolboxv2/mods/isaa/module.py

class Tools(MainTool, FileHandler):

    def __init__(self, app=None):

        self.run_callback = None
        # self.coding_projects: dict[str, ProjectManager] = {} # Assuming ProjectManager is defined elsewhere or removed
        if app is None:
            app = get_app("isaa-mod")
        self.version = version
        self.name = "isaa"
        self.Name = "isaa"
        self.color = "VIOLET2"
        self.config = {'controller-init': False,
                       'agents-name-list': [], # TODO Remain ComplexModel FastModel BlitzModel, AudioModel, (ImageModel[i/o], VideoModel[i/o]), SummaryModel
                       "FASTMODEL": os.getenv("FASTMODEL", "ollama/llama3.1"),
                       "AUDIOMODEL": os.getenv("AUDIOMODEL", "groq/whisper-large-v3-turbo"),
                       "BLITZMODEL": os.getenv("BLITZMODEL", "ollama/llama3.1"),
                       "COMPLEXMODEL": os.getenv("COMPLEXMODEL", "ollama/llama3.1"),
                       "SUMMARYMODEL": os.getenv("SUMMARYMODEL", "ollama/llama3.1"),
                       "IMAGEMODEL": os.getenv("IMAGEMODEL", "ollama/llama3.1"),
                       "DEFAULTMODELEMBEDDING": os.getenv("DEFAULTMODELEMBEDDING", "gemini/text-embedding-004"),
                       }
        self.per_data = {}
        self.agent_data: dict[str, dict] = {}  # Will store AgentConfig dicts
        self.keys = {
            "KEY": "key~~~~~~~",
            "Config": "config~~~~"
        }
        self.initstate = {}

        extra_path = ""
        if self.toolID:  # MainTool attribute
            extra_path = f"/{self.toolID}"
        self.observation_term_mem_file = f"{app.data_dir}/Memory{extra_path}/observationMemory/"
        self.config['controller_file'] = f"{app.data_dir}{extra_path}/controller.json"
        self.mas_text_summaries_dict = FileCache(folder=f"{app.data_dir}/Memory{extra_path}/summaries/")
        self.tools = {
            "name": "isaa",
            "Version": self.show_version,
            "mini_task_completion": self.mini_task_completion,
            "run_agent": self.run_agent,
            "save_to_mem": self.save_to_mem_sync,
            "get_agent": self.get_agent,
            "format_class": self.format_class,  # Now async
            "get_memory": self.get_memory,
            "save_all_memory_vis": self.save_all_memory_vis,
            "rget_mode": lambda mode: self.controller.rget(mode),
        }
        self.tools_interfaces: dict[str, ToolsInterface] = {}
        self.working_directory = os.getenv('ISAA_WORKING_PATH', os.getcwd())
        self.print_stream = stram_print
        self.global_stream_override = False  # Handled by FlowAgentBuilder
        self.lang_chain_tools_dict: dict[str, Any] = {}  # Store actual tool objects for wrapping

        self.agent_memory: AISemanticMemory = f"{app.id}{extra_path}/Memory"  # Path for AISemanticMemory
        self.controller = ControllerManager({})
        self.summarization_mode = 1
        self.summarization_limiter = 102000
        self.speak = lambda x, *args, **kwargs: x  # Placeholder

        self.default_setter = None  # For agent builder customization
        self.initialized = False

        FileHandler.__init__(self, f"isaa{extra_path.replace('/', '-')}.config", app.id if app else __name__)
        MainTool.__init__(self, load=self.on_start, v=self.version, tool=self.tools,
                          name=self.name, logs=None, color=self.color, on_exit=self.on_exit)

        from .extras.web_search import web_search
        async def web_search_tool(query: str) -> str:
            res = web_search(query)
            return await self.mas_text_summaries(str(res), min_length=12000, ref=query)
        self.web_search = web_search_tool
        self.shell_tool_function = shell_tool_function
        self.tools["shell"] = shell_tool_function

        self.print(f"Start {self.spec}.isaa")
        with Spinner(message="Starting module", symbols='c'):
            self.load_file_handler()
            config_fh = self.get_file_handler(self.keys["Config"])
            if config_fh is not None:
                if isinstance(config_fh, str):
                    try:
                        config_fh = json.loads(config_fh)
                    except json.JSONDecodeError:
                        self.print(f"Warning: Could not parse config from file handler: {config_fh[:100]}...")
                        config_fh = {}

                if isinstance(config_fh, dict):
                    # Merge, prioritizing existing self.config for defaults not in file
                    loaded_config = config_fh
                    for key, value in self.config.items():
                        if key not in loaded_config:
                            loaded_config[key] = value
                    self.config = loaded_config

            if self.spec == 'app':  # MainTool attribute
                self.load_keys_from_env()

            # Ensure directories exist
            Path(f"{get_app('isaa-initIsaa').data_dir}/Agents/").mkdir(parents=True, exist_ok=True)
            Path(f"{get_app('isaa-initIsaa').data_dir}/Memory/").mkdir(parents=True, exist_ok=True)

        #initialize_isaa_webui_module(self.app, self)
        #self.print("ISAA module started. fallback")

    def get_augment(self):
        # This needs to be adapted. Serialization of FlowAgent is through AgentConfig.
        return {
            "Agents": self.serialize_all(),  # Returns dict of AgentConfig dicts
        }

    async def init_from_augment(self, augment, agent_name: str = 'self'):
        """Initialize from augmented data using new builder system"""

        # Handle agent_name parameter
        if isinstance(agent_name, str):
            pass  # Use string name
        elif hasattr(agent_name, 'config'):  # FlowAgentBuilder
            agent_name = agent_name.config.name
        else:
            raise ValueError(f"Invalid agent_name type: {type(agent_name)}")

        a_keys = augment.keys()

        # Load agent configurations
        if "Agents" in a_keys:
            agents_configs_dict = augment['Agents']
            self.deserialize_all(agents_configs_dict)
            self.print("Agent configurations loaded.")

        # Tools are now handled by the builder system during agent creation
        if "tools" in a_keys:
            self.print("Tool configurations noted - will be applied during agent building")

    async def init_tools(self, tools_config: dict, agent_builder: FlowAgentBuilder):
        # This function needs to be adapted to add tools to the FlowAgentBuilder
        # For LangChain tools, they need to be wrapped as callables or ADK BaseTool instances.
        lc_tools_names = tools_config.get('lagChinTools', [])
        # hf_tools_names = tools_config.get('huggingTools', []) # HuggingFace tools are also LangChain tools
        # plugin_urls = tools_config.get('Plugins', [])

        all_lc_tool_names = list(set(lc_tools_names))  # + hf_tools_names

        for tool_name in all_lc_tool_names:
            try:
                # Load tool instance (LangChain's load_tools might return a list)
                loaded_tools = load_tools([tool_name], llm=None)  # LLM not always needed for tool definition
                for lc_tool_instance in loaded_tools:
                    # Wrap and add to builder
                    # Simple case: wrap lc_tool_instance.run or lc_tool_instance._run
                    if hasattr(lc_tool_instance, 'run') and callable(lc_tool_instance.run):
                        # ADK FunctionTool needs a schema, or infers it.
                        # We might need to manually create Pydantic models for args.
                        # For simplicity, assume ADK can infer or the tool takes simple args.
                        agent_builder.add_tool(lc_tool_instance.run, name=lc_tool_instance.name,
                                                             description=lc_tool_instance.description)
                        self.print(f"Added LangChain tool '{lc_tool_instance.name}' to builder.")
                        self.lang_chain_tools_dict[lc_tool_instance.name] = lc_tool_instance  # Store for reference
            except Exception as e:
                self.print(f"Failed to load/add LangChain tool '{tool_name}': {e}")

        # AIPluginTool needs more complex handling as it's a class
        # for url in plugin_urls:
        #     try:
        #         plugin = AIPluginTool.from_plugin_url(url)
        #         # Exposing AIPluginTool methods might require creating individual FunctionTools
        #         # Or creating a custom ADK BaseTool wrapper for AIPluginTool
        #         self.print(f"AIPluginTool {plugin.name} loaded. Manual ADK wrapping needed.")
        #     except Exception as e:
        #         self.print(f"Failed to load AIPlugin from {url}: {e}")

    def serialize_all(self):
        # Returns a copy of agent_data, which contains AgentConfig dicts
        # The exclude logic might be different if it was excluding fields from old AgentBuilder
        # For AgentConfig, exclusion happens during model_dump if needed.
        return copy.deepcopy(self.agent_data)

    def deserialize_all(self, data: dict[str, dict]):
        # Data is a dict of {agent_name: builder_config_dict}
        self.agent_data.update(data)
        # Clear instances from self.config so they are rebuilt with new configs
        for agent_name in data:
            self.config.pop(f'agent-instance-{agent_name}', None)

    async def init_isaa(self, name='self', build=False, **kwargs):
        if self.initialized:
            self.print(f"Already initialized. Getting agent/builder: {name}")
            # build=True implies getting the builder, build=False (default) implies getting agent instance
            return self.get_agent_builder(name) if build else await self.get_agent(name)

        self.initialized = True
        sys.setrecursionlimit(1500)
        self.load_keys_from_env()

        with Spinner(message="Building Controller", symbols='c'):
            self.controller.init(self.config['controller_file'])
        self.config["controller-init"] = True


        return self.get_agent_builder(name) if build else await self.get_agent(name)

    def show_version(self):
        self.print("Version: ", self.version)
        return self.version

    def on_start(self):

        initialize_isaa_webui_module(self.app, self)

        threading.Thread(target=self.load_to_mem_sync, daemon=True).start()
        self.print("ISAA module started.")

    def load_keys_from_env(self):
        # Update default model names from environment variables
        for key in self.config:
            if key.startswith("DEFAULTMODEL"):
                self.config[key] = os.getenv(key, self.config[key])
        self.config['VAULTS'] = os.getenv("VAULTS")

    def on_exit(self):
        self.app.run_bg_task_advanced(self.cleanup_tools_interfaces)
        # Save agent configurations
        for agent_name, agent_instance in self.config.items():
            if agent_name.startswith('agent-instance-') and agent_instance and isinstance(agent_instance, list) and isinstance(agent_instance[0], FlowAgent):
                self.app.run_bg_task_advanced(asyncio.gather(*[agent_instance.close() for agent_instance in agent_instance]))
                # If agent instance has its own save logic (e.g. cost tracker)
                # asyncio.run(agent_instance.close()) # This might block, consider task group
                # The AgentConfig is already in self.agent_data, which should be saved.
                pass  # Agent instances are not directly saved, their configs are.
        threading.Thread(target=self.save_to_mem_sync, daemon=True).start()  # Sync wrapper for save_to_mem

        # Save controller if initialized
        if self.config.get("controller-init"):
            self.controller.save(self.config['controller_file'])

        # Clean up self.config for saving
        clean_config = {}
        for key, value in self.config.items():
            if key.startswith('agent-instance-'): continue  # Don't save instances
            if key.startswith('LLM-model-'): continue  # Don't save langchain models
            clean_config[key] = value
        self.add_to_save_file_handler(self.keys["Config"], json.dumps(clean_config))

        # Save other persistent data
        self.save_file_handler()

    def save_to_mem_sync(self):
        # This used to call agent.save_memory(). FlowAgent does not have this.
        # If AISemanticMemory needs global saving, it should be handled by AISemanticMemory itself.
        # For now, this can be a no-op or save AISemanticMemory instances if managed by Tools.
        memory_instance = self.get_memory()  # Assuming this returns AISemanticMemory
        if hasattr(memory_instance, 'save_all_memories'):  # Hypothetical method
            memory_instance.save_all_memories(f"{get_app().data_dir}/Memory/")
        self.print("Memory saving process initiated")

    def load_to_mem_sync(self):
        # This used to call agent.save_memory(). FlowAgent does not have this.
        # If AISemanticMemory needs global saving, it should be handled by AISemanticMemory itself.
        # For now, this can be a no-op or save AISemanticMemory instances if managed by Tools.
        memory_instance = self.get_memory()  # Assuming this returns AISemanticMemory
        if hasattr(memory_instance, 'load_all_memories'):  # Hypothetical method
            memory_instance.load_all_memories(f"{get_app().data_dir}/Memory/")
        self.print("Memory loading process initiated")

    def get_agent_builder(self, name="self", extra_tools=None, add_tools=True, add_base_tools=True, working_directory=None) -> FlowAgentBuilder:
        if name == 'None':
            name = "self"

        if extra_tools is None:
            extra_tools = []

        self.print(f"Creating FlowAgentBuilder: {name}")

        # Create builder with agent-specific configuration
        config = AgentConfig(
            name=name,
            fast_llm_model=self.config.get(f'{name.upper()}MODEL', self.config['FASTMODEL']),
            complex_llm_model=self.config.get(f'{name.upper()}MODEL', self.config['COMPLEXMODEL']),
            system_message="You are a production-ready autonomous agent.",
            temperature=0.7,
            max_tokens_output=2048,
            max_tokens_input=32768,
            use_fast_response=True,
            max_parallel_tasks=3,
            verbose_logging=False
        )

        builder = FlowAgentBuilder(config=config)
        builder._isaa_ref = self  # Store ISAA reference

        # Load existing configuration if available
        agent_config_path = Path(f"{get_app().data_dir}/Agents/{name}/agent.json")
        if agent_config_path.exists():
            try:
                builder = FlowAgentBuilder.from_config_file(str(agent_config_path))
                builder._isaa_ref = self
                self.print(f"Loaded existing configuration for builder {name}")
            except Exception as e:
                self.print(f"Failed to load config for {name}: {e}. Using defaults.")

        # Apply global settings
        if self.global_stream_override:
            builder.verbose(True)

        # Apply custom setter if available
        if self.default_setter:
            builder = self.default_setter(builder, name)

        # Initialize ToolsInterface for this agent
        if not hasattr(self, 'tools_interfaces'):
            self.tools_interfaces = {}

        # Create or get existing ToolsInterface for this agent
        if name not in self.tools_interfaces:
            try:
                # Initialize ToolsInterface
                p = Path(get_app().data_dir) / "Agents" / name / "tools_session"
                p.mkdir(parents=True, exist_ok=True)
                tools_interface = ToolsInterface(
                    session_dir=str(Path(get_app().data_dir) / "Agents" / name / "tools_session"),
                    auto_remove=False,  # Keep session data for agents
                    variables={
                        'agent_name': name,
                        'isaa_instance': self
                    },
                    variable_manager=getattr(self, 'variable_manager', None),
                )
                if working_directory:
                    tools_interface.set_base_directory(working_directory)

                self.tools_interfaces[name] = tools_interface
                self.print(f"Created ToolsInterface for agent: {name}")

            except Exception as e:
                self.print(f"Failed to create ToolsInterface for {name}: {e}")
                self.tools_interfaces[name] = None

        tools_interface = self.tools_interfaces[name]

        # Add ISAA core tools
        async def run_isaa_agent_tool(target_agent_name: str, instructions: str, **kwargs_):
            if not instructions:
                return "No instructions provided."
            if target_agent_name.startswith('"') and target_agent_name.endswith('"') or target_agent_name.startswith(
                "'") and target_agent_name.endswith("'"):
                target_agent_name = target_agent_name[1:-1]
            return await self.run_agent(target_agent_name, text=instructions, **kwargs_)

        async def memory_search_tool(
            query: str,
            search_mode: str | None = "balanced",
            context_name: str | None = None
        ) -> str:
            """Memory search with configurable precision"""
            mem_instance = self.get_memory()
            memory_names_list = [name.strip() for name in context_name.split(',')] if context_name else None

            search_params = {
                "wide": {"k": 7, "min_similarity": 0.1, "cross_ref_depth": 3, "max_cross_refs": 4, "max_sentences": 8},
                "narrow": {"k": 2, "min_similarity": 0.75, "cross_ref_depth": 1, "max_cross_refs": 1,
                           "max_sentences": 3},
                "balanced": {"k": 3, "min_similarity": 0.2, "cross_ref_depth": 2, "max_cross_refs": 2,
                             "max_sentences": 5}
            }.get(search_mode,
                  {"k": 3, "min_similarity": 0.2, "cross_ref_depth": 2, "max_cross_refs": 2, "max_sentences": 5})

            return await mem_instance.query(
                query=query, memory_names=memory_names_list,
                query_params=search_params, to_str=True
            )

        async def save_to_memory_tool(data_to_save: str, context_name: str = name):
            mem_instance = self.get_memory()
            result = await mem_instance.add_data(context_name, str(data_to_save), direct=True)
            return 'Data added to memory.' if result else 'Error adding data to memory.'

        # Add ISAA core tools


        if add_base_tools:
            builder.add_tool(memory_search_tool, "memorySearch", "Search ISAA's semantic memory")
            builder.add_tool(save_to_memory_tool, "saveDataToMemory", "Save data to ISAA's semantic memory")
            builder.add_tool(self.web_search, "searchWeb", "Search the web for information")
            builder.add_tool(self.shell_tool_function, "shell", f"Run shell command in {detect_shell()}")

        # Add ToolsInterface tools dynamically
        if add_tools and tools_interface:
            try:
                # Get all tools from ToolsInterface
                interface_tools = tools_interface.get_tools()

                # Determine which tools to add based on agent name/type
                tool_categories = {
                    'code': ['execute_python', 'install_package'],
                    'file': ['write_file', 'replace_in_file', 'read_file', 'list_directory', 'create_directory'],
                    'session': ['get_execution_history', 'clear_session', 'get_variables'],
                    'config': ['set_base_directory', 'set_current_file']
                }

                # Determine which categories to include
                include_categories = set()
                name_lower = name.lower()

                # Code execution for development/coding agents
                if any(keyword in name_lower for keyword in ["dev", "code", "program", "script", "python", "rust", "worker"]):
                    include_categories.update(['code', 'file', 'session', 'config'])

                # Web tools for web-focused agents
                if any(keyword in name_lower for keyword in ["web", "browser", "scrape", "crawl", "extract"]):
                    include_categories.update(['file', 'session'])

                # File tools for file management agents
                if any(keyword in name_lower for keyword in ["file", "fs", "document", "write", "read"]):
                    include_categories.update(['file', 'session', 'config'])

                # Default: add core tools for general agents
                if not include_categories or name == "self":
                    include_categories.update(['code', 'file', 'session', 'config'])

                # Add selected tools
                tools_added = 0
                for tool_func, tool_name, tool_description in interface_tools:
                    # Check if this tool should be included
                    should_include = tool_name in extra_tools

                    if not should_include:
                        for category, tool_names in tool_categories.items():
                            if category in include_categories and tool_name in tool_names:
                                should_include = True
                                break

                    # Always include session management tools
                    if tool_name in ['get_execution_history', 'get_variables']:
                        should_include = True

                    if should_include:
                        try:
                            builder.add_tool(tool_func, tool_name, tool_description)
                            tools_added += 1
                        except Exception as e:
                            self.print(f"Failed to add tool {tool_name}: {e}")

                self.print(f"Added {tools_added} ToolsInterface tools to agent {name}")

            except Exception as e:
                self.print(f"Error adding ToolsInterface tools to {name}: {e}")

        # Configure cost tracking
        builder.with_budget_manager(max_cost=100.0)

        # Store agent configuration
        try:
            agent_dir = Path(f"{get_app().data_dir}/Agents/{name}")
            agent_dir.mkdir(parents=True, exist_ok=True)

            # Save agent metadata
            metadata = {
                'name': name,
                'created_at': time.time(),
                'tools_interface_available': tools_interface is not None,
                'session_dir': str(agent_dir / "tools_session")
            }

            metadata_file = agent_dir / "metadata.json"
            with open(metadata_file, 'w') as f:
                json.dump(metadata, f, indent=2)

        except Exception as e:
            self.print(f"Failed to save agent metadata for {name}: {e}")

        return builder

    def get_tools_interface(self, agent_name: str = "self") -> ToolsInterface | None:
        """
        Get the ToolsInterface instance for a specific agent.

        Args:
            agent_name: Name of the agent

        Returns:
            ToolsInterface instance or None if not found
        """
        if not hasattr(self, 'tools_interfaces'):
            return None

        return self.tools_interfaces.get(agent_name)

    async def configure_tools_interface(self, agent_name: str, **kwargs) -> bool:
        """
        Configure the ToolsInterface for a specific agent.

        Args:
            agent_name: Name of the agent
            **kwargs: Configuration parameters

        Returns:
            True if successful, False otherwise
        """
        tools_interface = self.get_tools_interface(agent_name)
        if not tools_interface:
            self.print(f"No ToolsInterface found for agent {agent_name}")
            return False

        try:
            # Configure based on provided parameters
            if 'base_directory' in kwargs:
                await tools_interface.set_base_directory(kwargs['base_directory'])

            if 'current_file' in kwargs:
                await tools_interface.set_current_file(kwargs['current_file'])

            if 'variables' in kwargs:
                tools_interface.ipython.user_ns.update(kwargs['variables'])

            self.print(f"Configured ToolsInterface for agent {agent_name}")
            return True

        except Exception as e:
            self.print(f"Failed to configure ToolsInterface for {agent_name}: {e}")
            return False

    async def cleanup_tools_interfaces(self):
        """
        Cleanup all ToolsInterface instances.
        """
        if not hasattr(self, 'tools_interfaces'):
            return

        async def cleanup_async():
            for name, tools_interface in self.tools_interfaces.items():
                if tools_interface:
                    try:
                        await tools_interface.__aexit__(None, None, None)
                    except Exception as e:
                        self.print(f"Error cleaning up ToolsInterface for {name}: {e}")

        # Run cleanup
        try:
            await cleanup_async()
            self.tools_interfaces.clear()
            self.print("Cleaned up all ToolsInterface instances")
        except Exception as e:
            self.print(f"Error during ToolsInterface cleanup: {e}")

    async def register_agent(self, agent_builder: FlowAgentBuilder):
        agent_name = agent_builder.config.name

        if f'agent-instance-{agent_name}' in self.config:
            self.print(f"Agent '{agent_name}' instance already exists. Overwriting config and rebuilding on next get.")
            self.config.pop(f'agent-instance-{agent_name}', None)

        # Save the builder's configuration
        config_path = Path(f"{get_app().data_dir}/Agents/{agent_name}/agent.json")
        agent_builder.save_config(str(config_path), format='json')
        self.print(f"Saved FlowAgentBuilder config for '{agent_name}' to {config_path}")

        # Store serializable config in agent_data
        self.agent_data[agent_name] = agent_builder.config.model_dump()

        if agent_name not in self.config.get("agents-name-list", []):
            if "agents-name-list" not in self.config:
                self.config["agents-name-list"] = []
            self.config["agents-name-list"].append(agent_name)

        self.print(f"FlowAgent '{agent_name}' configuration registered. Will be built on first use.")
        row_agent_builder_sto[agent_name] = agent_builder  # Cache builder

    async def get_agent(self, agent_name="Normal", model_override: str | None = None) -> FlowAgent:
        if "agents-name-list" not in self.config:
            self.config["agents-name-list"] = []

        instance_key = f'agent-instance-{agent_name}'
        if instance_key in self.config:
            agent_instance = self.config[instance_key]
            if model_override and agent_instance.amd.fast_llm_model != model_override:
                self.print(f"Model override for {agent_name}: {model_override}. Rebuilding.")
                self.config.pop(instance_key, None)
            else:
                self.print(f"Returning existing FlowAgent instance: {agent_name}")
                return agent_instance

        builder_to_use = None

        # Try to get cached builder first
        if agent_name in row_agent_builder_sto:
            builder_to_use = row_agent_builder_sto[agent_name]
            self.print(f"Using cached builder for {agent_name}")

        # Try to load from stored config
        elif agent_name in self.agent_data:
            self.print(f"Loading configuration for FlowAgent: {agent_name}")
            try:
                config = AgentConfig(**self.agent_data[agent_name])
                builder_to_use = FlowAgentBuilder(config=config)
            except Exception as e:
                self.print(f"Error loading config for {agent_name}: {e}. Falling back to default.")

        # Create default builder if none found
        if builder_to_use is None:
            self.print(f"No existing config for {agent_name}. Creating default builder.")
            builder_to_use = self.get_agent_builder(agent_name)

        # Apply overrides and ensure correct name
        builder_to_use._isaa_ref = self
        if model_override:
            builder_to_use.with_models(model_override, model_override)

        if builder_to_use.config.name != agent_name:
            builder_to_use.with_name(agent_name)

        self.print(
            f"Building FlowAgent: {agent_name} with models {builder_to_use.config.fast_llm_model} - {builder_to_use.config.complex_llm_model}")

        # Build the agent
        agent_instance: FlowAgent = await builder_to_use.build()

        if agent_instance.amd.name == "self":
            self.app.run_bg_task_advanced(agent_instance.initialize_context_awareness)

        if interface := self.get_tools_interface(agent_name):
            interface.variable_manager = agent_instance.variable_manager

        # colletive cabability cahring for reduched reduanda analysis _tool_capabilities
        agent_tool_nams = set(agent_instance.tool_registry.keys())

        tools_data = {}
        for _agent_name in self.config["agents-name-list"]:
            _instance_key = f'agent-instance-{_agent_name}'
            if _instance_key not in self.config:
                if agent_name != "self" and _agent_name == "self":
                    await self.get_agent("self")

            if _instance_key not in self.config:
                continue
            _agent_instance = self.config[_instance_key]
            _agent_tool_nams = set(_agent_instance._tool_capabilities.keys())
            # extract the tool names that are in both agents_registry
            overlap_tool_nams = agent_tool_nams.intersection(_agent_tool_nams)
            _tc = _agent_instance._tool_capabilities
            for tool_name in overlap_tool_nams:
                if tool_name not in _tc:
                    continue
                tools_data[tool_name] = _tc[tool_name]

        agent_instance._tool_capabilities.update(tools_data)
        # Cache the instance and update tracking
        self.config[instance_key] = agent_instance
        if agent_name not in self.agent_data:
            self.agent_data[agent_name] = builder_to_use.config.model_dump()
        if agent_name not in self.config["agents-name-list"]:
            self.config["agents-name-list"].append(agent_name)

        self.print(f"Built and cached FlowAgent instance: {agent_name}")
        return agent_instance

    @export(api=True, version=version, request_as_kwarg=True, mod_name="isaa")
    async def mini_task_completion(self, mini_task: str | None = None, user_task: str | None = None, mode: Any = None,  # LLMMode
                                   max_tokens_override: int | None = None, task_from="system",
                                   stream_function: Callable | None = None, message_history: list | None = None, agent_name="TaskCompletion", use_complex: bool = False, request: RequestData | None = None, form_data: dict | None = None, data: dict | None = None, **kwargs):
        if request is not None or form_data is not None or data is not None:
            data_dict = (request.request.body if request else None) or form_data or data
            mini_task = mini_task or  data_dict.get("mini_task")
            user_task = user_task or data_dict.get("user_task")
            mode = mode or data_dict.get("mode")
            max_tokens_override = max_tokens_override or data_dict.get("max_tokens_override")
            task_from = data_dict.get("task_from") or task_from
            agent_name = data_dict.get("agent_name") or agent_name
            use_complex = use_complex or data_dict.get("use_complex")
            kwargs = kwargs or data_dict.get("kwargs")
            message_history = message_history or data_dict.get("message_history")
            if isinstance(message_history, str):
                message_history = json.loads(message_history)
        print(mini_task, agent_name, use_complex, kwargs, message_history, form_data or data)
        if mini_task is None: return None
        if agent_name is None: return None
        if mini_task == "test": return "test"
        self.print(f"Running mini task, volume {len(mini_task)}")

        agent = await self.get_agent(agent_name)  # Ensure agent is retrieved (and built if needed)

        effective_system_message = agent.amd.system_message
        if mode and hasattr(mode, 'system_msg') and mode.system_msg:
            effective_system_message = mode.system_msg

        messages = []
        if effective_system_message:
            messages.append({"role": "system", "content": effective_system_message})
        if message_history:
            messages.extend(message_history)

        current_prompt = mini_task
        if user_task:  # If user_task is provided, it becomes the main prompt, mini_task is context
            messages.append({"role": task_from, "content": mini_task})  # mini_task as prior context
            current_prompt = user_task  # user_task as the current prompt

        messages.append({"role": "user", "content": current_prompt})

        # Prepare params for a_run_llm_completion
        if use_complex:
            llm_params = {"model": agent.amd.complex_llm_model, "messages": messages}
        else:
            llm_params = {"model": agent.amd.fast_llm_model if agent.amd.use_fast_response else agent.amd.complex_llm_model, "messages": messages}
        if max_tokens_override:
            llm_params['max_tokens'] = max_tokens_override
        else:
            llm_params['max_tokens'] = agent.amd.max_tokens
        if kwargs:
            llm_params.update(kwargs)  # Add any additional kwargs
        if stream_function:
            llm_params['stream'] = True
            # FlowAgent a_run_llm_completion handles stream_callback via agent.stream_callback
            # For a one-off, we might need a temporary override or pass it if supported.
            # For now, assume stream_callback is set on agent instance if needed globally.
            # If stream_function is for this call only, agent.a_run_llm_completion needs modification
            # or we use a temporary agent instance. This part is tricky.
            # Let's assume for now that if stream_function is passed, it's a global override for this agent type.
            original_stream_cb = agent.stream_callback
            original_stream_val = agent.stream
            agent.stream_callback = stream_function
            agent.stream = True
            try:
                response_content = await agent.a_run_llm_completion(**llm_params)
            finally:
                agent.stream_callback = original_stream_cb
                agent.stream = original_stream_val  # Reset to builder's config
            return response_content  # Streaming output handled by callback

        llm_params['stream'] = False
        response_content = await agent.a_run_llm_completion(**llm_params)
        return response_content

    async def mini_task_completion_format(self, mini_task, format_schema: type[BaseModel],
                                          max_tokens_override: int | None = None, agent_name="TaskCompletion",
                                          task_from="system", mode_overload: Any = None, user_task: str | None = None, auto_context=False, **kwargs):
        if mini_task is None: return None
        self.print(f"Running formatted mini task, volume {len(mini_task)}")

        agent = await self.get_agent(agent_name)

        effective_system_message = None
        if mode_overload and hasattr(mode_overload, 'system_msg') and mode_overload.system_msg:
            effective_system_message = mode_overload.system_msg

        message_context = []
        if effective_system_message:
            message_context.append({"role": "system", "content": effective_system_message})

        current_prompt = mini_task
        if user_task:
            message_context.append({"role": task_from, "content": mini_task})
            current_prompt = user_task

        # Use agent.a_format_class
        try:
            result_dict = await agent.a_format_class(
                pydantic_model=format_schema,
                prompt=current_prompt,
                message_context=message_context,
                auto_context=auto_context
                # max_tokens can be part of agent's model config or passed if a_format_class supports it
            )
            if format_schema == bool:  # Special handling for boolean schema
                # a_format_class returns a dict, e.g. {"value": True}. Extract the bool.
                # This depends on how bool schema is defined. A common way: class BoolResponse(BaseModel): value: bool
                return result_dict.get("value", False) if isinstance(result_dict, dict) else False
            return result_dict
        except Exception as e:
            self.print(f"Error in mini_task_completion_format: {e}")
            return None  # Or raise

    @export(api=True, version=version, name="version")
    async def get_version(self, *a,**k):
        return self.version

    @export(api=True, version=version, request_as_kwarg=True, mod_name="isaa")
    async def format_class(self, format_schema: type[BaseModel] | None = None, task: str | None = None, agent_name="TaskCompletion", auto_context=False, request: RequestData | None = None, form_data: dict | None = None, data: dict | None = None, **kwargs):
        if request is not None or form_data is not None or data is not None:
            data_dict = (request.request.body if request else None) or form_data or data
            format_schema = format_schema or data_dict.get("format_schema")
            task = task or data_dict.get("task")
            agent_name = data_dict.get("agent_name") or agent_name
            auto_context = auto_context or data_dict.get("auto_context")
            kwargs = kwargs or data_dict.get("kwargs")
        if format_schema is None or not task: return None
        agent = None
        if isinstance(agent_name, str):
            agent = await self.get_agent(agent_name)
        elif isinstance(agent_name, FlowAgent):
            agent = agent_name
        else:
            raise TypeError("agent_name must be str or FlowAgent instance")

        return await agent.a_format_class(format_schema, task, auto_context=auto_context)

    async def run_agent(self, name: str | FlowAgent,
                        text: str,
                        verbose: bool = False,  # Handled by agent's own config mostly
                        session_id: str | None = None,
                        progress_callback: Callable[[Any], None | Awaitable[None]] | None = None,
                        **kwargs):  # Other kwargs for a_run
        if text is None: return ""
        if name is None: return ""
        if text == "test": return ""

        agent_instance = None
        if isinstance(name, str):
            agent_instance = await self.get_agent(name)
        elif isinstance(name, FlowAgent):
            agent_instance = name
        else:
            return self.return_result().default_internal_error(
                f"Invalid agent identifier type: {type(name)}")

        self.print(f"Running agent {agent_instance.amd.name} for task: {text[:100]}...")
        save_p = None
        if progress_callback:
            save_p = agent_instance.progress_callback
            agent_instance.progress_callback = progress_callback

        if verbose:
            agent_instance.verbose = True

        # Call FlowAgent's a_run method
        response = await agent_instance.a_run(
            query=text,
            session_id=session_id,
            user_id=None,
            stream_callback=None

        )
        if save_p:
            agent_instance.progress_callback = save_p

        return response

    # mass_text_summaries and related methods remain complex and depend on AISemanticMemory
    # and specific summarization strategies. For now, keeping their structure,
    # but calls to self.format_class or self.mini_task_completion will become async.

    async def mas_text_summaries(self, text, min_length=36000, ref=None, max_tokens_override=None):
        len_text = len(text)
        if len_text < min_length: return text
        key = self.one_way_hash(text, 'summaries', 'isaa')
        value = self.mas_text_summaries_dict.get(key)
        if value is not None: return value

        # This part needs to become async due to format_class
        # Simplified version:
        from .extras.modes import (
            SummarizationMode,
            # crate_llm_function_from_langchain_tools,
        )
        summary = await self.mini_task_completion(
            mini_task=f"Summarize this text, focusing on aspects related to '{ref if ref else 'key details'}'. The text is: {text}",
            mode=self.controller.rget(SummarizationMode), max_tokens_override=max_tokens_override, agent_name="self")

        if summary is None or not isinstance(summary, str):
            # Fallback or error handling
            summary = text[:min_length] + "... (summarization failed)"

        self.mas_text_summaries_dict.set(key, summary)
        return summary

    def get_memory(self, name: str | None = None) -> AISemanticMemory:
        # This method's logic seems okay, AISemanticMemory is a separate system.
        logger_ = get_logger()  # Renamed to avoid conflict with self.logger
        if isinstance(self.agent_memory, str):  # Path string
            logger_.info(Style.GREYBG("AISemanticMemory Initialized from path"))
            self.agent_memory = AISemanticMemory(base_path=self.agent_memory)

        cm = self.agent_memory
        if name is not None:
            # Assuming AISemanticMemory.get is synchronous or you handle async appropriately
            # If AISemanticMemory methods become async, this needs adjustment
            mem_kb = cm.get(name)  # This might return a list of KnowledgeBase or single one
            return mem_kb
        return cm

    async def save_all_memory_vis(self, dir_path=None):
        if dir_path is None:
            dir_path = f"{get_app().data_dir}/Memory/vis"
            Path(dir_path).mkdir(parents=True, exist_ok=True)
        self.load_to_mem_sync()
        for name, kb in self.get_memory().memories.items():
            self.print(f"Saving to {name}.html with {len(kb.concept_extractor.concept_graph.concepts)} concepts")
            await kb.vis(output_file=f"{dir_path}/{name}.html")
        return dir_path

    async def host_agent_ui(
        self,
        agent,
        host: str = "0.0.0.0",
        port: int | None = None,
        access: str = 'local',
        registry_server: str | None = None,
        public_name: str | None = None,
        description: str | None = None,
        use_builtin_server: bool = None
    ) -> dict[str, str]:
        """
        Unified agent hosting with WebSocket-enabled UI and optional registry publishing.

        Args:
            agent: Agent or Chain instance to host
            host: Host address (default: 0.0.0.0 for remote access)
            port: Port number (auto-assigned if None)
            access: 'local', 'remote', or 'registry'
            registry_server: Registry server URL for publishing (e.g., "ws://localhost:8080/ws/registry/connect")
            public_name: Public name for registry publishing
            description: Description for registry publishing
            use_builtin_server: Use toolbox built-in server vs standalone Python server

        Returns:
            Dictionary with access URLs and configuration
        """
        use_builtin_server = use_builtin_server or self.app.is_server
        if not hasattr(self, '_hosted_agents'):
            self._hosted_agents = {}

        agent_id = f"agent_{secrets.token_urlsafe(8)}"

        # Generate unique port if not specified
        if not port:
            port = 8765 + len(self._hosted_agents)

        # Store agent reference
        self._hosted_agents[agent_id] = {
            'agent': agent,
            'port': port,
            'host': host,
            'access': access,
            'public_name': public_name or f"Agent_{agent_id}",
            'description': description
        }

        result = {
            'agent_id': agent_id,
            'local_url': f"http://{host}:{port}",
            'status': 'starting'
        }

        if use_builtin_server:
            # Use toolbox built-in server
            result.update(await self._setup_builtin_server_hosting(agent_id, agent, host, port))
        else:
            # Use standalone Python server
            result.update(await self._setup_standalone_server_hosting(agent_id, agent, host, port))

        # Handle registry publishing if requested
        if access in ['remote', 'registry'] and registry_server:
            if not public_name:
                raise ValueError("public_name required for registry publishing")

            registry_result = await self._publish_to_registry(
                agent=agent,
                public_name=public_name,
                registry_server=registry_server,
                description=description,
                agent_id=agent_id
            )
            result.update(registry_result)

        self.app.print(f"🚀 Agent '{result.get('public_name', agent_id)}' hosted successfully!")
        self.app.print(f"   Local UI: {result['local_url']}")
        if 'public_url' in result:
            self.app.print(f"   Public URL: {result['public_url']}")
            self.app.print(f"   API Key: {result.get('api_key', 'N/A')}")

        return result

    # toolboxv2/mods/isaa/__init__.py - Missing Methods

    import asyncio
    import json
    import secrets
    import threading
    import time
    from concurrent.futures import ThreadPoolExecutor
    from http.server import BaseHTTPRequestHandler, HTTPServer
    from urllib.parse import parse_qs, urlparse


    async def _handle_reset_context(self, agent_id: str, agent, conn_id: str):
        """Handle context reset requests from WebSocket UI."""

        try:
            # Reset agent context if supported
            if hasattr(agent, 'clear_context'):
                agent.clear_context()
                message = "Context reset successfully"
                success = True
            else:
                message = "Agent does not support context reset"
                success = False

            # Send response back to UI
            await self._broadcast_to_agent_ui(agent_id, {
                'event': 'reset_response',
                'data': {
                    'success': success,
                    'message': message,
                    'timestamp': time.time()
                }
            })

            self.app.print(f"Context reset requested for agent {agent_id}: {message}")

        except Exception as e:
            error_message = f"Context reset failed: {str(e)}"
            self.app.print(f"Context reset error for agent {agent_id}: {e}")

            await self._broadcast_to_agent_ui(agent_id, {
                'event': 'error',
                'data': {
                    'error': error_message,
                    'timestamp': time.time()
                }
            })

    async def _handle_get_status(self, agent_id: str, agent, conn_id: str):
        """Handle status requests from WebSocket UI."""

        try:
            # Collect agent status information
            status_info = {
                'agent_id': agent_id,
                'agent_name': getattr(agent, 'name', 'Unknown'),
                'agent_type': agent.__class__.__name__,
                'status': 'active',
                'timestamp': time.time(),
                'server_type': 'builtin'
            }

            # Add additional status if available
            if hasattr(agent, 'status'):
                try:
                    agent_status = agent.status()
                    if isinstance(agent_status, dict):
                        status_info.update(agent_status)
                except:
                    pass

            # Add hosted agent info
            if hasattr(self, '_hosted_agents') and agent_id in self._hosted_agents:
                hosted_info = self._hosted_agents[agent_id]
                status_info.update({
                    'host': hosted_info.get('host'),
                    'port': hosted_info.get('port'),
                    'access': hosted_info.get('access'),
                    'public_name': hosted_info.get('public_name')
                })

            # Send status back to UI
            await self._broadcast_to_agent_ui(agent_id, {
                'event': 'status_response',
                'data': status_info
            })

            self.app.print(f"Status requested for agent {agent_id}")

        except Exception as e:
            error_message = f"Status retrieval failed: {str(e)}"
            self.app.print(f"Status error for agent {agent_id}: {e}")

            await self._broadcast_to_agent_ui(agent_id, {
                'event': 'error',
                'data': {
                    'error': error_message,
                    'timestamp': time.time()
                }
            })


    async def stop_hosted_agent(self, agent_id: str = None, port: int = None):
        """Stop a hosted agent by agent_id or port."""

        if not hasattr(self, '_hosted_agents') and not hasattr(self, '_standalone_servers'):
            self.app.print("No hosted agents found")
            return False

        # Stop by agent_id
        if agent_id:
            if hasattr(self, '_hosted_agents') and agent_id in self._hosted_agents:
                agent_info = self._hosted_agents[agent_id]
                agent_port = agent_info.get('port')

                # Stop standalone server if exists
                if hasattr(self, '_standalone_servers') and agent_port in self._standalone_servers:
                    server_info = self._standalone_servers[agent_port]
                    try:
                        server_info['server'].shutdown()
                        self.app.print(f"Stopped standalone server for agent {agent_id}")
                    except:
                        pass

                # Clean up hosted agent info
                del self._hosted_agents[agent_id]
                self.app.print(f"Stopped hosted agent {agent_id}")
                return True

        # Stop by port
        if port:
            if hasattr(self, '_standalone_servers') and port in self._standalone_servers:
                server_info = self._standalone_servers[port]
                try:
                    server_info['server'].shutdown()
                    self.app.print(f"Stopped server on port {port}")
                    return True
                except Exception as e:
                    self.app.print(f"Failed to stop server on port {port}: {e}")
                    return False

        self.app.print("Agent or port not found")
        return False

    async def list_hosted_agents(self) -> dict[str, Any]:
        """List all currently hosted agents."""

        hosted_info = {
            'builtin_agents': {},
            'standalone_agents': {},
            'total_count': 0
        }

        # Built-in server agents
        if hasattr(self, '_hosted_agents'):
            for agent_id, info in self._hosted_agents.items():
                hosted_info['builtin_agents'][agent_id] = {
                    'public_name': info.get('public_name'),
                    'host': info.get('host'),
                    'port': info.get('port'),
                    'access': info.get('access'),
                    'description': info.get('description')
                }

        # Standalone server agents
        if hasattr(self, '_standalone_servers'):
            for port, info in self._standalone_servers.items():
                hosted_info['standalone_agents'][info['agent_id']] = {
                    'port': port,
                    'thread_alive': info['thread'].is_alive(),
                    'server_type': 'standalone'
                }

        hosted_info['total_count'] = len(hosted_info['builtin_agents']) + len(hosted_info['standalone_agents'])

        return hosted_info

    def _create_agent_ws_connect_handler(self, agent_id: str):
        """Create WebSocket connect handler for specific agent."""

        async def on_connect(app, conn_id: str, session: dict):
            if not hasattr(self, '_agent_connections'):
                self._agent_connections = {}

            if agent_id not in self._agent_connections:
                self._agent_connections[agent_id] = set()

            self._agent_connections[agent_id].add(conn_id)

            # Send initial status
            await app.ws_send(conn_id, {
                'event': 'agent_connected',
                'data': {
                    'agent_id': agent_id,
                    'status': 'ready',
                    'capabilities': ['chat', 'progress_tracking', 'real_time_updates']
                }
            })

            self.app.print(f"UI client connected to agent {agent_id}: {conn_id}")

        return on_connect

    def _create_agent_ws_message_handler(self, agent_id: str, agent):
        """Create WebSocket message handler for specific agent."""

        async def on_message(app, conn_id: str, session: dict, payload: dict):
            event = payload.get('event')
            data = payload.get('data', {})

            if event == 'chat_message':
                await self._handle_chat_message(agent_id, agent, conn_id, data)
            elif event == 'reset_context':
                await self._handle_reset_context(agent_id, agent, conn_id)
            elif event == 'get_status':
                await self._handle_get_status(agent_id, agent, conn_id)
            else:
                self.app.print(f"Unknown event from UI: {event}")

        return on_message

    def _create_agent_ws_disconnect_handler(self, agent_id: str):
        """Create WebSocket disconnect handler for specific agent."""

        async def on_disconnect(app, conn_id: str, session: dict = None):
            if hasattr(self, '_agent_connections') and agent_id in self._agent_connections:
                self._agent_connections[agent_id].discard(conn_id)

            self.app.print(f"UI client disconnected from agent {agent_id}: {conn_id}")

        return on_disconnect


    async def _broadcast_to_agent_ui(self, agent_id: str, message: dict):
        """Broadcast message to all UI clients connected to specific agent."""
        if not hasattr(self, '_agent_connections') or agent_id not in self._agent_connections:
            return

        for conn_id in self._agent_connections[agent_id].copy():
            try:
                await self.app.ws_send(conn_id, message)
            except Exception as e:
                self.app.print(f"Failed to send to UI client {conn_id}: {e}")
                self._agent_connections[agent_id].discard(conn_id)

    async def _publish_to_registry(
        self,
        agent,
        public_name: str,
        registry_server: str,
        description: str | None = None,
        agent_id: str | None = None
    ) -> dict[str, str]:
        """Publish agent to registry server."""
        try:
            # Import registry client dynamically to avoid circular imports
            registry_client_module = __import__("toolboxv2.mods.registry.client", fromlist=["get_registry_client"])
            get_registry_client = registry_client_module.get_registry_client

            client = get_registry_client(self.app)

            # Connect if not already connected
            if not client.ws or not client.ws.open:
                await client.connect(registry_server)

            if not client.ws or not client.ws.open:
                raise Exception("Failed to connect to registry server")

            # Register the agent
            reg_info = await client.register(agent, public_name, description)

            if reg_info:
                return {
                    'public_url': reg_info.public_url,
                    'api_key': reg_info.public_api_key,
                    'public_agent_id': reg_info.public_agent_id,
                    'registry_status': 'published'
                }
            else:
                raise Exception("Registration failed")

        except Exception as e:
            self.app.print(f"Registry publishing failed: {e}")
            return {'registry_status': 'failed', 'registry_error': str(e)}

    def _get_enhanced_agent_ui_html(self, agent_id: str) -> str:
        """Get production-ready enhanced UI HTML with comprehensive progress visualization."""
        agent_info = self._hosted_agents.get(agent_id, {})
        server_info = {
            'server_type': 'standalone' if not hasattr(self.app, 'tb') else 'builtin',
            'agent_id': agent_id
        }

        # Update the JavaScript section in the HTML template:
        js_config = f"""
                window.SERVER_CONFIG = {json.dumps(server_info)};
            """
        html_template = """<!DOCTYPE html>
    <html lang="en">
    <head>
        <meta charset="UTF-8">
        <meta name="viewport" content="width=device-width, initial-scale=1.0">
        <title>{agent_name}</title>
        <script src="https://cdn.jsdelivr.net/npm/marked/marked.min.js"></script>
        <style>
            :root {
                --bg-primary: #0d1117;
                --bg-secondary: #161b22;
                --bg-tertiary: #21262d;
                --text-primary: #f0f6fc;
                --text-secondary: #8b949e;
                --text-muted: #6e7681;
                --accent-blue: #58a6ff;
                --accent-green: #3fb950;
                --accent-red: #f85149;
                --accent-orange: #d29922;
                --accent-purple: #a5a5f5;
                --accent-cyan: #39d0d8;
                --border-color: #30363d;
                --shadow: 0 2px 8px rgba(0, 0, 0, 0.3);
            }

            * { margin: 0; padding: 0; box-sizing: border-box; }

            body {
                font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Helvetica, Arial, sans-serif;
                background: var(--bg-primary);
                color: var(--text-primary);
                height: 100vh;
                display: flex;
                flex-direction: column;
                overflow: hidden;
            }

            .header {
                background: var(--bg-tertiary);
                padding: 12px 20px;
                border-bottom: 1px solid var(--border-color);
                display: flex;
                align-items: center;
                justify-content: space-between;
                box-shadow: var(--shadow);
                z-index: 100;
            }

            .agent-info {
                display: flex;
                align-items: center;
                gap: 16px;
            }

            .agent-title {
                font-size: 18px;
                font-weight: 600;
                color: var(--accent-blue);
            }

            .agent-status {
                display: flex;
                align-items: center;
                gap: 8px;
                font-size: 14px;
            }

            .status-dot {
                width: 10px;
                height: 10px;
                border-radius: 50%;
                background: var(--accent-red);
                animation: pulse 2s infinite;
            }

            .status-dot.connected {
                background: var(--accent-green);
                animation: none;
            }

            .status-dot.processing {
                background: var(--accent-orange);
                animation: pulse 1s infinite;
            }

            @keyframes pulse {
                0%, 100% { opacity: 1; }
                50% { opacity: 0.5; }
            }

            .main-container {
                display: grid;
                grid-template-columns: 2fr 1.5fr 1fr;
                grid-template-rows: 1fr 1fr;
                grid-template-areas:
                    "chat outline activity"
                    "chat system graph";
                flex: 1;
                gap: 1px;
                background: var(--border-color);
                overflow: hidden;
            }

            .panel {
                background: var(--bg-secondary);
                display: flex;
                flex-direction: column;
                overflow: hidden;
            }

            .chat-panel { grid-area: chat; }
            .outline-panel { grid-area: outline; }
            .activity-panel { grid-area: activity; }
            .system-panel { grid-area: system; }
            .graph-panel { grid-area: graph; }

            .panel-header {
                padding: 12px 16px;
                background: var(--bg-tertiary);
                border-bottom: 1px solid var(--border-color);
                font-weight: 600;
                font-size: 12px;
                text-transform: uppercase;
                letter-spacing: 0.5px;
                display: flex;
                align-items: center;
                gap: 8px;
            }

            .panel-content {
                flex: 1;
                overflow-y: auto;
                padding: 12px;
            }

            /* Chat Panel Styles */
            .chat-messages {
                flex: 1;
                overflow-y: auto;
                padding: 16px;
                display: flex;
                flex-direction: column;
                gap: 16px;
            }

            .message {
                display: flex;
                align-items: flex-start;
                gap: 12px;
                max-width: 85%;
            }

            .message.user {
                flex-direction: row-reverse;
                margin-left: auto;
            }

            .message-avatar {
                width: 32px;
                height: 32px;
                border-radius: 50%;
                display: flex;
                align-items: center;
                justify-content: center;
                font-size: 12px;
                font-weight: 600;
                flex-shrink: 0;
            }

            .message.user .message-avatar {
                background: var(--accent-blue);
            }

            .message.agent .message-avatar {
                background: var(--accent-green);
            }

            .message-content {
                padding: 12px 16px;
                border-radius: 12px;
                line-height: 1.5;
                font-size: 14px;
            }

            .message.user .message-content {
                background: var(--accent-blue);
                color: white;
            }

            .message.agent .message-content {
                background: var(--bg-tertiary);
                border: 1px solid var(--border-color);
            }

            .chat-input-area {
                border-top: 1px solid var(--border-color);
                padding: 16px;
                display: flex;
                gap: 12px;
            }

            .chat-input {
                flex: 1;
                background: var(--bg-primary);
                border: 1px solid var(--border-color);
                border-radius: 8px;
                padding: 12px;
                color: var(--text-primary);
                font-size: 14px;
            }

            .chat-input:focus {
                outline: none;
                border-color: var(--accent-blue);
            }

            .send-button {
                background: var(--accent-blue);
                color: white;
                border: none;
                border-radius: 8px;
                padding: 12px 20px;
                cursor: pointer;
                font-weight: 600;
                transition: all 0.2s;
            }

            .send-button:hover:not(:disabled) {
                background: #4493f8;
                transform: translateY(-1px);
            }

            .send-button:disabled {
                opacity: 0.5;
                cursor: not-allowed;
                transform: none;
            }

            /* Progress Indicator */
            .progress-indicator {
                display: none;
                align-items: center;
                gap: 12px;
                padding: 12px 16px;
                background: var(--bg-tertiary);
                border-top: 1px solid var(--border-color);
                font-size: 14px;
            }

            .progress-indicator.active { display: flex; }

            .spinner {
                width: 16px;
                height: 16px;
                border: 2px solid var(--border-color);
                border-top: 2px solid var(--accent-blue);
                border-radius: 50%;
                animation: spin 1s linear infinite;
            }

            @keyframes spin {
                0% { transform: rotate(0deg); }
                100% { transform: rotate(360deg); }
            }

            /* Outline Panel Styles */
            .outline-progress {
                margin-bottom: 16px;
            }

            .outline-header {
                display: flex;
                align-items: center;
                justify-content: space-between;
                margin-bottom: 12px;
            }

            .outline-title {
                font-weight: 600;
                color: var(--accent-cyan);
            }

            .outline-stats {
                font-size: 12px;
                color: var(--text-muted);
            }

            .progress-bar {
                width: 100%;
                height: 6px;
                background: var(--bg-primary);
                border-radius: 3px;
                overflow: hidden;
                margin-bottom: 16px;
            }

            .progress-fill {
                height: 100%;
                background: linear-gradient(90deg, var(--accent-blue), var(--accent-cyan));
                width: 0%;
                transition: width 0.5s ease;
            }

            .outline-steps {
                display: flex;
                flex-direction: column;
                gap: 8px;
            }

            .outline-step {
                display: flex;
                align-items: center;
                gap: 10px;
                padding: 8px 12px;
                border-radius: 6px;
                background: var(--bg-primary);
                border-left: 3px solid var(--border-color);
                transition: all 0.3s;
            }

            .outline-step.active {
                border-left-color: var(--accent-orange);
                background: rgba(217, 153, 34, 0.1);
            }

            .outline-step.completed {
                border-left-color: var(--accent-green);
                background: rgba(63, 185, 80, 0.1);
            }

            .step-icon {
                font-size: 14px;
                width: 16px;
            }

            .step-text {
                flex: 1;
                font-size: 13px;
            }

            .step-method {
                font-size: 11px;
                color: var(--text-muted);
                background: var(--bg-tertiary);
                padding: 2px 6px;
                border-radius: 4px;
            }

            /* Activity Panel Styles */
            .current-activity {
                background: var(--bg-primary);
                border: 1px solid var(--border-color);
                border-radius: 6px;
                padding: 12px;
                margin-bottom: 12px;
            }

            .activity-header {
                display: flex;
                align-items: center;
                gap: 8px;
                margin-bottom: 8px;
            }

            .activity-title {
                font-weight: 600;
                color: var(--accent-orange);
            }

            .activity-duration {
                font-size: 11px;
                color: var(--text-muted);
                background: var(--bg-tertiary);
                padding: 2px 6px;
                border-radius: 4px;
            }

            .activity-description {
                font-size: 13px;
                line-height: 1.4;
                color: var(--text-secondary);
            }

            .meta-tools-list {
                display: flex;
                flex-direction: column;
                gap: 6px;
            }

            .meta-tool {
                display: flex;
                align-items: center;
                gap: 8px;
                padding: 6px 10px;
                background: var(--bg-primary);
                border-radius: 4px;
                font-size: 12px;
            }

            .tool-icon {
                width: 12px;
                text-align: center;
            }

            .tool-name {
                flex: 1;
                color: var(--text-secondary);
            }

            .tool-status {
                font-size: 10px;
                padding: 2px 6px;
                border-radius: 3px;
            }

            .tool-status.running {
                background: var(--accent-orange);
                color: white;
            }

            .tool-status.completed {
                background: var(--accent-green);
                color: white;
            }

            .tool-status.error {
                background: var(--accent-red);
                color: white;
            }

            /* System Panel Styles */
            .system-grid {
                display: grid;
                grid-template-columns: 1fr 2fr;
                gap: 8px 12px;
                font-size: 12px;
            }

            .system-key {
                color: var(--text-muted);
                font-weight: 500;
            }

            .system-value {
                color: var(--text-primary);
                font-family: 'SF Mono', Monaco, monospace;
                word-break: break-word;
            }

            .current-node {
                background: var(--bg-primary);
                padding: 8px 10px;
                border-radius: 6px;
                margin-bottom: 12px;
                border: 1px solid var(--border-color);
            }

            .node-name {
                font-weight: 600;
                color: var(--accent-purple);
                margin-bottom: 4px;
            }

            .node-operation {
                font-size: 11px;
                color: var(--text-muted);
            }

            /* Graph Panel Styles */
            .agent-graph {
                display: flex;
                flex-direction: column;
                align-items: center;
                gap: 8px;
                padding: 8px;
            }

            .graph-node {
                padding: 6px 12px;
                background: var(--bg-primary);
                border: 1px solid var(--border-color);
                border-radius: 6px;
                font-size: 11px;
                text-align: center;
                min-width: 80px;
            }

            .graph-node.active {
                border-color: var(--accent-orange);
                background: rgba(217, 153, 34, 0.1);
            }

            .graph-node.completed {
                border-color: var(--accent-green);
                background: rgba(63, 185, 80, 0.1);
            }

            .graph-arrow {
                color: var(--text-muted);
                font-size: 12px;
            }

            /* Connection Error Styles */
            .connection-error {
                background: var(--accent-red);
                color: white;
                padding: 8px 12px;
                margin: 8px;
                border-radius: 6px;
                font-size: 12px;
                text-align: center;
            }

            .fallback-mode {
                background: var(--accent-orange);
                color: white;
                padding: 8px 12px;
                margin: 8px;
                border-radius: 6px;
                font-size: 12px;
                text-align: center;
            }
        </style>
    </head>
    <body>
        <div class="header">
            <div class="agent-info">
                <div class="agent-title">{agent_name}</div>
                <div class="text-secondary">{agent_description}</div>
            </div>
            <div class="agent-status">
                <div class="status-dot" id="status-dot"></div>
                <span id="status-text">Initializing...</span>
            </div>
        </div>

        <div class="main-container">
            <!-- Chat Panel -->
            <div class="panel chat-panel">
                <div class="panel-header">💬 Conversation</div>
                <div class="chat-messages" id="chat-messages">
                    <div class="message agent">
                        <div class="message-avatar">AI</div>
                        <div class="message-content">Hello! I'm ready to help you. What would you like to know?</div>
                    </div>
                </div>
                <div class="progress-indicator" id="progress-indicator">
                    <div class="spinner"></div>
                    <span id="progress-text">Processing...</span>
                </div>
                <div class="chat-input-area">
                    <input type="text" id="chat-input" class="chat-input" placeholder="Type your message...">
                    <button id="send-button" class="send-button">Send</button>
                </div>
            </div>

            <!-- Outline & Progress Panel -->
            <div class="panel outline-panel">
                <div class="panel-header">📋 Execution Outline</div>
                <div class="panel-content">
                    <div class="outline-progress">
                        <div class="outline-header">
                            <div class="outline-title" id="outline-title">Ready</div>
                            <div class="outline-stats" id="outline-stats">0/0 steps</div>
                        </div>
                        <div class="progress-bar">
                            <div class="progress-fill" id="outline-progress-fill"></div>
                        </div>
                    </div>
                    <div class="outline-steps" id="outline-steps">
                        <div class="outline-step">
                            <div class="step-icon">⏳</div>
                            <div class="step-text">Waiting for query...</div>
                        </div>
                    </div>
                    <div class="current-activity" id="current-activity" style="display: none;">
                        <div class="activity-header">
                            <div class="activity-title" id="activity-title">Current Activity</div>
                            <div class="activity-duration" id="activity-duration">0s</div>
                        </div>
                        <div class="activity-description" id="activity-description"></div>
                    </div>
                </div>
            </div>

            <!-- Activity & Meta-Tools Panel -->
            <div class="panel activity-panel">
                <div class="panel-header">⚙️ Meta-Tool Activity</div>
                <div class="panel-content">
                    <div class="meta-tools-list" id="meta-tools-list">
                        <div style="color: var(--text-muted); font-size: 12px; text-align: center; padding: 20px;">
                            No activity yet
                        </div>
                    </div>
                </div>
            </div>

            <!-- System Status Panel -->
            <div class="panel system-panel">
                <div class="panel-header">🔧 System Status</div>
                <div class="panel-content">
                    <div class="current-node" id="current-node">
                        <div class="node-name" id="node-name">System</div>
                        <div class="node-operation" id="node-operation">Idle</div>
                    </div>
                    <div class="system-grid" id="system-grid">
                        <div class="system-key">Status</div>
                        <div class="system-value">Ready</div>
                        <div class="system-key">Runtime</div>
                        <div class="system-value">0s</div>
                        <div class="system-key">Events</div>
                        <div class="system-value">0</div>
                        <div class="system-key">Errors</div>
                        <div class="system-value">0</div>
                    </div>
                </div>
            </div>

            <!-- Agent Graph Panel -->
            <div class="panel graph-panel">
                <div class="panel-header">🌐 Agent Flow</div>
                <div class="panel-content">
                    <div class="agent-graph" id="agent-graph">
                        <div class="graph-node">LLMReasonerNode</div>
                        <div class="graph-arrow">↓</div>
                        <div class="graph-node">Ready</div>
                    </div>
                </div>
            </div>
        </div>

        <script unSave="true">
            __SERVER_CONFIG__
            class ProductionAgentUI {
                constructor() {
                    this.ws = null;
                    this.isProcessing = false;
                    this.sessionId = 'ui_session_' + Math.random().toString(36).substr(2, 9);
                    this.startTime = null;
                    this.reconnectAttempts = 0;
                    this.maxReconnectAttempts = 10;
                    this.reconnectDelay = 1000;
                    this.useWebSocket = true;
                    this.fallbackMode = false;

                    // Progress tracking
                    this.currentOutline = null;
                    this.currentActivity = null;
                    this.metaTools = new Map();
                    this.systemStatus = {};
                    this.agentGraph = [];
                    this.progressEvents = [];

                    this.elements = {
                        statusDot: document.getElementById('status-dot'),
                        statusText: document.getElementById('status-text'),
                        chatMessages: document.getElementById('chat-messages'),
                        chatInput: document.getElementById('chat-input'),
                        sendButton: document.getElementById('send-button'),
                        progressIndicator: document.getElementById('progress-indicator'),
                        progressText: document.getElementById('progress-text'),

                        // Outline elements
                        outlineTitle: document.getElementById('outline-title'),
                        outlineStats: document.getElementById('outline-stats'),
                        outlineProgressFill: document.getElementById('outline-progress-fill'),
                        outlineSteps: document.getElementById('outline-steps'),
                        currentActivity: document.getElementById('current-activity'),
                        activityTitle: document.getElementById('activity-title'),
                        activityDuration: document.getElementById('activity-duration'),
                        activityDescription: document.getElementById('activity-description'),

                        // Meta-tools elements
                        metaToolsList: document.getElementById('meta-tools-list'),

                        // System elements
                        currentNode: document.getElementById('current-node'),
                        nodeName: document.getElementById('node-name'),
                        nodeOperation: document.getElementById('node-operation'),
                        systemGrid: document.getElementById('system-grid'),

                        // Graph elements
                        agentGraph: document.getElementById('agent-graph')
                    };
                    this.init();
                }


                init() {

                    this.configureAPIPaths();
                    this.setupEventListeners();
                    this.detectServerMode();
                    this.startStatusUpdates();
                }

                configureAPIPaths() {
                    const serverType = window.SERVER_CONFIG?.server_type || 'standalone';

                    if (serverType === 'builtin') {
                        this.apiPaths = {
                            status: '/api/agent_ui/status',
                            run: '/api/agent_ui/run_agent',
                            reset: '/api/agent_ui/reset_context'
                        };
                        this.useWebSocket = true;
                    } else {
                        this.apiPaths = {
                            status: '/api/status',
                            run: '/api/run',
                            reset: '/api/reset'
                        };
                        this.useWebSocket = false;
                        this.enableFallbackMode();
                    }
                }

                setupEventListeners() {
                    this.elements.sendButton.addEventListener('click', () => this.sendMessage());
                    this.elements.chatInput.addEventListener('keypress', (e) => {
                        if (e.key === 'Enter' && !this.isProcessing) {
                            this.sendMessage();
                        }
                    });

                    // Handle page visibility for reconnection
                    document.addEventListener('visibilitychange', () => {
                        if (!document.hidden && (!this.ws || this.ws.readyState === WebSocket.CLOSED)) {
                            this.connectWebSocket();
                        }
                    });
                }

                detectServerMode() {
                    // Use configured paths instead of hardcoded ones
                    fetch(this.apiPaths.status)
                        .then(response => response.json())
                        .then(data => {
                            this.addLogEntry(`Server detected: ${data.server_type || 'standalone'}`, 'info');
                            if (data.server_type === 'builtin' && this.useWebSocket) {
                                this.connectWebSocket();
                            }
                        })
                        .catch(() => {
                            this.addLogEntry('Server detection failed, using fallback mode', 'error');
                            this.enableFallbackMode();
                        });
                }

                connectWebSocket() {
                    if (!this.useWebSocket) return;

                    try {
                        // Construct WebSocket URL more robustly
                        const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
                        const wsUrl = `${protocol}//${window.location.host}/ws/agent_ui/connect`;

                        this.addLogEntry(`Attempting WebSocket connection to: ${wsUrl}`);
                        this.ws = new WebSocket(wsUrl);

                        this.ws.onopen = () => {
                            this.reconnectAttempts = 0;
                            this.fallbackMode = false;
                            this.setStatus('connected', 'Connected');
                            this.addLogEntry('WebSocket connected successfully', 'success');
                            this.removeFallbackIndicators();
                        };

                        this.ws.onmessage = (event) => {
                            try {
                                const message = JSON.parse(event.data);
                                this.handleWebSocketMessage(message);
                            } catch (error) {
                                this.addLogEntry(`WebSocket message parse error: ${error.message}`, 'error');
                            }
                        };

                        this.ws.onclose = (event) => {
                            this.setStatus('disconnected', 'Disconnected');
                            this.addLogEntry(`WebSocket disconnected (code: ${event.code})`, 'error');
                            this.scheduleReconnection();
                        };

                        this.ws.onerror = (error) => {
                            this.setStatus('error', 'Connection Error');
                            this.addLogEntry('WebSocket connection error', 'error');
                            this.scheduleReconnection();
                        };

                    } catch (error) {
                        this.addLogEntry(`WebSocket setup error: ${error.message}`, 'error');
                        this.enableFallbackMode();
                    }
                }

                scheduleReconnection() {
                    if (this.reconnectAttempts >= this.maxReconnectAttempts) {
                        this.addLogEntry('Max reconnection attempts reached, enabling fallback mode', 'error');
                        this.enableFallbackMode();
                        return;
                    }

                    this.reconnectAttempts++;
                    const delay = Math.min(this.reconnectDelay * this.reconnectAttempts, 30000);

                    this.setStatus('error', `Reconnecting in ${delay/1000}s (attempt ${this.reconnectAttempts})`);

                    setTimeout(() => {
                        if (!this.ws || this.ws.readyState === WebSocket.CLOSED) {
                            this.connectWebSocket();
                        }
                    }, delay);
                }

                enableFallbackMode() {
                    this.fallbackMode = true;
                    this.useWebSocket = false;
                    this.setStatus('disconnected', 'Fallback Mode (API Only)');
                    this.showFallbackIndicator();
                    this.addLogEntry('WebSocket unavailable - using API fallback mode', 'info');
                }

                showFallbackIndicator() {
                    const indicator = document.createElement('div');
                    indicator.className = 'fallback-mode';
                    indicator.textContent = 'Using API fallback mode - limited real-time updates';
                    indicator.id = 'fallback-indicator';
                    document.body.appendChild(indicator);
                }

                removeFallbackIndicators() {
                    const indicator = document.getElementById('fallback-indicator');
                    if (indicator) {
                        indicator.remove();
                    }
                }

                handleWebSocketMessage(message) {
                    try {
                        switch (message.event) {
                            case 'agent_connected':
                                this.addLogEntry('Agent ready for interaction', 'success');
                                this.updateSystemStatus({
                                    status: 'Connected',
                                    capabilities: message.data.capabilities
                                });
                                break;

                            case 'processing_start':
                                this.setProcessing(true);
                                this.startTime = Date.now();
                                this.addLogEntry(`Processing: ${message.data.query}`, 'progress');
                                this.resetProgressTracking();
                                break;

                            case 'progress_update':
                                this.handleProgressUpdate(message.data);
                                break;

                            case 'outline_update':
                                this.handleOutlineUpdate(message.data);
                                break;

                            case 'meta_tool_update':
                                this.handleMetaToolUpdate(message.data);
                                break;

                            case 'activity_update':
                                this.handleActivityUpdate(message.data);
                                break;

                            case 'system_update':
                                this.handleSystemUpdate(message.data);
                                break;

                            case 'graph_update':
                                this.handleGraphUpdate(message.data);
                                break;

                            case 'chat_response':
                                this.addMessage('agent', message.data.response);
                                this.setProcessing(false);
                                this.addLogEntry('Response completed', 'success');
                                this.showFinalSummary(message.data);
                                break;

                            case 'error':
                                this.addMessage('agent', `Error: ${message.data.error}`);
                                this.setProcessing(false);
                                this.addLogEntry(`Error: ${message.data.error}`, 'error');
                                break;

                            default:
                                console.log('Unhandled WebSocket message:', message);
                        }
                    } catch (error) {
                        this.addLogEntry(`Message handling error: ${error.message}`, 'error');
                    }
                }

                handleProgressUpdate(data) {
                    this.progressEvents.push(data);

                    const progressText = `${data.event_type}: ${data.status || 'processing'}`;
                    this.elements.progressText.textContent = progressText;

                    // Update based on event type
                    if (data.event_type === 'reasoning_loop') {
                        this.addLogEntry(`🧠 Reasoning loop #${data.loop_number || '?'}`, 'reasoning');
                        this.updateCurrentActivity({
                            title: 'Reasoning',
                            description: data.current_focus || 'Deep thinking in progress',
                            duration: data.time_in_activity || 0
                        });
                    } else if (data.event_type === 'meta_tool_call') {
                        this.addLogEntry(`⚙️ Meta-tool: ${data.meta_tool_name || 'unknown'}`, 'meta-tool');
                    } else {
                        this.addLogEntry(`Progress - ${progressText}`, 'progress');
                    }

                    // Update system status
                    this.updateSystemStatus({
                        current_node: data.node_name,
                        current_operation: data.event_type,
                        runtime: this.getRuntime(),
                        events: this.progressEvents.length
                    });
                }

                handleOutlineUpdate(data) {
                    this.currentOutline = data;

                    if (data.outline_created && data.steps) {
                        this.elements.outlineTitle.textContent = 'Execution Outline';

                        const completedCount = (data.completed_steps || []).length;
                        const totalCount = data.total_steps || data.steps.length;

                        this.elements.outlineStats.textContent = `${completedCount}/${totalCount} steps`;

                        // Update progress bar
                        const progress = totalCount > 0 ? (completedCount / totalCount) * 100 : 0;
                        this.elements.outlineProgressFill.style.width = `${progress}%`;

                        // Update steps
                        this.updateOutlineSteps(data.steps, data.current_step, data.completed_steps || []);

                        this.addLogEntry(`Outline progress: ${completedCount}/${totalCount} steps completed`, 'outline');
                    }
                }

                updateOutlineSteps(steps, currentStep, completedSteps) {
                    this.elements.outlineSteps.innerHTML = '';

                    steps.forEach((step, index) => {
                        const stepEl = document.createElement('div');
                        stepEl.className = 'outline-step';

                        const stepId = step.id || (index + 1);
                        let icon = '⏳';

                        if (completedSteps.includes(stepId)) {
                            stepEl.classList.add('completed');
                            icon = '✅';
                        } else if (stepId === currentStep) {
                            stepEl.classList.add('active');
                            icon = '🔄';
                        }

                        stepEl.innerHTML = `
                            <div class="step-icon">${icon}</div>
                            <div class="step-text">${step.description || `Step ${stepId}`}</div>
                            <div class="step-method">${step.method || 'unknown'}</div>
                        `;

                        this.elements.outlineSteps.appendChild(stepEl);
                    });
                }

                handleMetaToolUpdate(data) {
                    const toolId = `${data.meta_tool_name}_${Date.now()}`;
                    const toolData = {
                        name: data.meta_tool_name,
                        status: data.status || 'running',
                        timestamp: Date.now(),
                        phase: data.execution_phase,
                        data: data
                    };

                    this.metaTools.set(toolId, toolData);
                    this.updateMetaToolsList();

                    // Add to log with appropriate icon
                    const statusIcon = data.status === 'completed' ? '✅' :
                                     data.status === 'error' ? '❌' : '⚙️';
                    this.addLogEntry(`${statusIcon} ${data.meta_tool_name}: ${data.status || 'running'}`, 'meta-tool');
                }

                updateMetaToolsList() {
                    this.elements.metaToolsList.innerHTML = '';

                    if (this.metaTools.size === 0) {
                        this.elements.metaToolsList.innerHTML = `
                            <div style="color: var(--text-muted); font-size: 12px; text-align: center; padding: 20px;">
                                No meta-tool activity yet
                            </div>
                        `;
                        return;
                    }

                    // Show recent meta-tools (last 8)
                    const recentTools = Array.from(this.metaTools.values())
                        .sort((a, b) => b.timestamp - a.timestamp)
                        .slice(0, 8);

                    recentTools.forEach(tool => {
                        const toolEl = document.createElement('div');
                        toolEl.className = 'meta-tool';

                        const icons = {
                            internal_reasoning: '🧠',
                            delegate_to_llm_tool_node: '🎯',
                            create_and_execute_plan: '📋',
                            manage_internal_task_stack: '📚',
                            advance_outline_step: '➡️',
                            write_to_variables: '💾',
                            read_from_variables: '📖',
                            direct_response: '✨'
                        };

                        const icon = icons[tool.name] || '⚙️';
                        const displayName = tool.name.replace(/_/g, ' ');
                        const age = Math.floor((Date.now() - tool.timestamp) / 1000);

                        toolEl.innerHTML = `
                            <div class="tool-icon">${icon}</div>
                            <div class="tool-name">${displayName} (${age}s ago)</div>
                            <div class="tool-status ${tool.status}">${tool.status}</div>
                        `;

                        this.elements.metaToolsList.appendChild(toolEl);
                    });
                }

                handleActivityUpdate(data) {
                    this.currentActivity = data;
                    this.updateCurrentActivity(data);
                }

                updateCurrentActivity(data) {
                    if (data.primary_activity && data.primary_activity !== 'Unknown') {
                        this.elements.currentActivity.style.display = 'block';
                        this.elements.activityTitle.textContent = data.primary_activity || data.title;

                        const duration = data.time_in_current_activity || data.duration || 0;
                        if (duration > 0) {
                            this.elements.activityDuration.textContent = this.formatDuration(duration);
                        }

                        this.elements.activityDescription.textContent =
                            data.detailed_description || data.description || '';
                    } else {
                        this.elements.currentActivity.style.display = 'none';
                    }
                }

                handleSystemUpdate(data) {
                    this.systemStatus = { ...this.systemStatus, ...data };
                    this.updateSystemStatus(data);
                }

                updateSystemStatus(data) {
                    // Update current node
                    if (data.current_node) {
                        this.elements.nodeName.textContent = data.current_node;
                        this.elements.nodeOperation.textContent = data.current_operation || 'Processing';
                    }

                    // Update system grid
                    const gridData = [
                        ['Status', data.status || this.systemStatus.status || 'Running'],
                        ['Runtime', this.formatDuration(data.runtime || this.getRuntime())],
                        ['Events', data.events || this.progressEvents.length],
                        ['Errors', data.error_count || this.systemStatus.error_count || 0],
                        ['Node', data.current_node || this.systemStatus.current_node || 'Unknown']
                    ];

                    if (data.total_cost !== undefined) {
                        gridData.push(['Cost', `$${data.total_cost.toFixed(4)}`]);
                    }

                    if (data.total_tokens !== undefined) {
                        gridData.push(['Tokens', data.total_tokens.toLocaleString()]);
                    }

                    this.elements.systemGrid.innerHTML = '';
                    gridData.forEach(([key, value]) => {
                        this.elements.systemGrid.innerHTML += `
                            <div class="system-key">${key}</div>
                            <div class="system-value">${value}</div>
                        `;
                    });
                }

                handleGraphUpdate(data) {
                    this.agentGraph = data.nodes || [];
                    this.updateAgentGraph();
                }

                updateAgentGraph() {
                    this.elements.agentGraph.innerHTML = '';

                    if (this.agentGraph.length === 0) {
                        const currentNode = this.systemStatus.current_node || 'LLMReasonerNode';
                        this.elements.agentGraph.innerHTML = `
                            <div class="graph-node active">${currentNode}</div>
                            <div class="graph-arrow">↓</div>
                            <div class="graph-node">Processing</div>
                        `;
                        return;
                    }

                    this.agentGraph.forEach((node, index) => {
                        const nodeEl = document.createElement('div');
                        nodeEl.className = 'graph-node';

                        if (node.active) nodeEl.classList.add('active');
                        if (node.completed) nodeEl.classList.add('completed');

                        nodeEl.textContent = node.name || `Node ${index + 1}`;
                        this.elements.agentGraph.appendChild(nodeEl);

                        if (index < this.agentGraph.length - 1) {
                            const arrow = document.createElement('div');
                            arrow.className = 'graph-arrow';
                            arrow.textContent = '↓';
                            this.elements.agentGraph.appendChild(arrow);
                        }
                    });
                }

                async sendMessage() {
                    const message = this.elements.chatInput.value.trim();
                    if (!message || this.isProcessing) return;

                    this.addMessage('user', message);
                    this.elements.chatInput.value = '';

                    if (this.useWebSocket && this.ws && this.ws.readyState === WebSocket.OPEN) {
                        // Send via WebSocket
                        this.ws.send(JSON.stringify({
                            event: 'chat_message',
                            data: {
                                message: message,
                                session_id: this.sessionId
                            }
                        }));
                    } else {
                        // Fallback to API
                        await this.sendMessageViaAPI(message);
                    }
                }

                async sendMessageViaAPI(message) {
                    this.setProcessing(true);
                    this.startTime = Date.now();
                    this.resetProgressTracking();

                    try {
                        const response = await fetch(this.apiPaths.run, {
                            method: 'POST',
                            headers: {
                                'Content-Type': 'application/json'
                            },
                            body: JSON.stringify({
                                query: message,
                                session_id: this.sessionId,
                                include_progress: true
                            })
                        });

                        const result = await response.json();

                        if (result.success) {
                            this.addMessage('agent', result.result);
                            this.addLogEntry(`Request completed via API`, 'success');

                            // Process progress events if available
                            if (result.progress_events) {
                                this.processAPIProgressEvents(result.progress_events);
                            }

                            // Process enhanced progress if available
                            if (result.enhanced_progress) {
                                this.processEnhancedProgress(result.enhanced_progress);
                            }
                        } else {
                            this.addMessage('agent', `Error: ${result.error}`);
                            this.addLogEntry(`API request failed: ${result.error}`, 'error');
                        }

                    } catch (error) {
                        this.addMessage('agent', `Network error: ${error.message}`);
                        this.addLogEntry(`Network error: ${error.message}`, 'error');
                    } finally {
                        this.setProcessing(false);
                    }
                }

                processAPIProgressEvents(events) {
                    events.forEach(event => {
                        this.handleProgressUpdate(event);
                    });
                }

                processEnhancedProgress(progress) {
                    if (progress.outline) {
                        this.handleOutlineUpdate(progress.outline);
                    }
                    if (progress.activity) {
                        this.handleActivityUpdate(progress.activity);
                    }
                    if (progress.system) {
                        this.handleSystemUpdate(progress.system);
                    }
                    if (progress.graph) {
                        this.handleGraphUpdate(progress.graph);
                    }
                }

                resetProgressTracking() {
                    this.progressEvents = [];
                    this.metaTools.clear();
                    this.updateSystemStatus({ status: 'Processing', events: 0 });
                }

                showFinalSummary(data) {
                    if (data.final_summary) {
                        const summary = data.final_summary;
                        this.addLogEntry(`Final Summary - Outline: ${summary.outline_completed ? 'Complete' : 'Partial'}, Meta-tools: ${summary.total_meta_tools}, Nodes: ${summary.total_nodes}`, 'success');
                    }
                }

                addMessage(sender, content) {
                    const messageEl = document.createElement('div');
                    messageEl.classList.add('message', sender);

                    const avatarEl = document.createElement('div');
                    avatarEl.classList.add('message-avatar');
                    avatarEl.textContent = sender === 'user' ? 'You' : 'AI';

                    const contentEl = document.createElement('div');
                    contentEl.classList.add('message-content');

                    if (sender === 'agent' && window.marked) {
                        try {
                            contentEl.innerHTML = marked.parse(content);
                        } catch (error) {
                            contentEl.textContent = content;
                        }
                    } else {
                        contentEl.textContent = content;
                    }

                    messageEl.appendChild(avatarEl);
                    messageEl.appendChild(contentEl);

                    this.elements.chatMessages.appendChild(messageEl);
                    this.elements.chatMessages.scrollTop = this.elements.chatMessages.scrollHeight;
                }

                addLogEntry(message, type = 'info') {
                    // For debugging - could show in a log panel
                    const timestamp = new Date().toLocaleTimeString();
                    console.log(`[${timestamp}] [${type.toUpperCase()}] ${message}`);
                }

                setStatus(status, text) {
                    this.elements.statusDot.className = `status-dot ${status}`;
                    this.elements.statusText.textContent = text;
                }

                setProcessing(processing) {
                    this.isProcessing = processing;
                    this.elements.sendButton.disabled = processing;
                    this.elements.chatInput.disabled = processing;

                    if (processing) {
                        this.elements.progressIndicator.classList.add('active');
                        this.setStatus('processing', 'Processing');
                    } else {
                        this.elements.progressIndicator.classList.remove('active');
                        this.setStatus(this.ws && this.ws.readyState === WebSocket.OPEN ? 'connected' : 'disconnected',
                                      this.ws && this.ws.readyState === WebSocket.OPEN ? 'Connected' : 'Disconnected');
                        this.startTime = null;
                    }
                }

                formatDuration(seconds) {
                    if (typeof seconds !== 'number') return '0s';
                    if (seconds < 60) return `${seconds.toFixed(1)}s`;
                    if (seconds < 3600) return `${Math.floor(seconds/60)}m${Math.floor(seconds%60)}s`;
                    return `${Math.floor(seconds/3600)}h${Math.floor((seconds%3600)/60)}m`;
                }

                getRuntime() {
                    return this.startTime ? (Date.now() - this.startTime) / 1000 : 0;
                }

                startStatusUpdates() {
                    setInterval(() => {
                        if (this.isProcessing) {
                            this.updateSystemStatus({ runtime: this.getRuntime() });
                        }
                    }, 1000);
                }
            }

            // Initialize the production UI
            if (!window.TB) {

                document.addEventListener('DOMContentLoaded', () => {
                    window.agentUI = new ProductionAgentUI();
                });
            } else {
                TB.once(() => {
                    window.agentUI = new ProductionAgentUI();
                });
            }
        </script>
    </body>
    </html>"""

        return (html_template.
                replace("{agent_name}", agent_info.get('public_name', 'Agent Interface')).
                replace("{agent_description}", agent_info.get('description', '')).
                replace("__SERVER_CONFIG__", js_config)
                )

    async def _handle_chat_message_with_progress_integration(self, agent_id: str, agent, conn_id: str, data: dict):
        """Enhanced chat message handler with ProgressiveTreePrinter integration."""
        query = data.get('message', '')
        session_id = data.get('session_id', f"ui_session_{conn_id}")

        if not query:
            return

        # Create ProgressiveTreePrinter for real-time UI updates
        from toolboxv2.mods.isaa.extras.terminal_progress import (
            ProgressiveTreePrinter,
            VerbosityMode,
        )
        progress_printer = ProgressiveTreePrinter(
            mode=VerbosityMode.STANDARD,
            use_rich=False,
            auto_refresh=False
        )

        # Enhanced progress callback that extracts all UI data
        async def comprehensive_progress_callback(event):
            try:
                # Add event to progress printer for processing
                progress_printer.tree_builder.add_event(event)

                # Get comprehensive summary from the printer
                summary = progress_printer.tree_builder.get_execution_summary()

                # Extract outline information
                outline_info = progress_printer._get_current_outline_info()

                # Extract current activity
                activity_info = progress_printer._get_detailed_current_activity()

                # Extract tool usage
                tool_usage = progress_printer._get_tool_usage_summary()

                # Extract task progress
                task_progress = progress_printer._get_task_executor_progress()

                # Send basic progress update
                await self._broadcast_to_agent_ui(agent_id, {
                    'event': 'progress_update',
                    'data': {
                        'event_type': event.event_type,
                        'status': getattr(event, 'status', 'processing').value if hasattr(event, 'status') and event.status else 'unknown',
                        'node_name': getattr(event, 'node_name', 'Unknown'),
                        'timestamp': event.timestamp,
                        'loop_number': getattr(event.metadata, {}).get('reasoning_loop', 0),
                        'meta_tool_name': getattr(event.metadata, {}).get('meta_tool_name'),
                        'current_focus': getattr(event.metadata, {}).get('current_focus', ''),
                        'time_in_activity': activity_info.get('time_in_current_activity', 0)
                    }
                })

                # Send outline updates
                if outline_info.get('outline_created'):
                    await self._broadcast_to_agent_ui(agent_id, {
                        'event': 'outline_update',
                        'data': outline_info
                    })

                # Send meta-tool updates
                if event.metadata and event.metadata.get('meta_tool_name'):
                    await self._broadcast_to_agent_ui(agent_id, {
                        'event': 'meta_tool_update',
                        'data': {
                            'meta_tool_name': event.metadata['meta_tool_name'],
                            'status': 'completed' if event.success else (
                                'error' if event.success is False else 'running'),
                            'execution_phase': event.metadata.get('execution_phase', 'unknown'),
                            'reasoning_loop': event.metadata.get('reasoning_loop', 0),
                            'timestamp': event.timestamp
                        }
                    })

                # Send activity updates
                if activity_info['primary_activity'] != 'Unknown':
                    await self._broadcast_to_agent_ui(agent_id, {
                        'event': 'activity_update',
                        'data': activity_info
                    })

                # Send system updates
                await self._broadcast_to_agent_ui(agent_id, {
                    'event': 'system_update',
                    'data': {
                        'current_node': summary['execution_flow']['current_node'],
                        'current_operation': activity_info.get('primary_activity', 'Processing'),
                        'status': 'Processing',
                        'runtime': summary['timing']['elapsed'],
                        'total_events': summary['performance_metrics']['total_events'],
                        'error_count': summary['performance_metrics']['error_count'],
                        'total_cost': summary['performance_metrics']['total_cost'],
                        'total_tokens': summary['performance_metrics']['total_tokens'],
                        'completed_nodes': summary['session_info']['completed_nodes'],
                        'total_nodes': summary['session_info']['total_nodes'],
                        'tool_usage': {
                            'tools_used': list(tool_usage.get('tools_used', set())),
                            'tools_active': list(tool_usage.get('tools_active', set())),
                            'current_tool_operation': tool_usage.get('current_tool_operation')
                        }
                    }
                })

                # Send graph updates
                flow_nodes = []
                for node_name in summary['execution_flow']['flow']:
                    if node_name in progress_printer.tree_builder.nodes:
                        node = progress_printer.tree_builder.nodes[node_name]
                        flow_nodes.append({
                            'name': node_name,
                            'active': node_name in summary['execution_flow']['active_nodes'],
                            'completed': (node.status.value == 'completed') if node.status else False,
                            'status': node.status.value if node.status else 'unknown'
                        })

                if flow_nodes:
                    await self._broadcast_to_agent_ui(agent_id, {
                        'event': 'graph_update',
                        'data': {'nodes': flow_nodes}
                    })

            except Exception as e:
                self.app.print(f"Comprehensive progress callback error: {e}")

        # Set progress callback
        original_callback = getattr(agent, 'progress_callback', None)

        try:
            if hasattr(agent, 'set_progress_callback'):
                agent.set_progress_callback(comprehensive_progress_callback)
            elif hasattr(agent, 'progress_callback'):
                agent.progress_callback = comprehensive_progress_callback

            # Send processing start notification
            await self._broadcast_to_agent_ui(agent_id, {
                'event': 'processing_start',
                'data': {'query': query, 'session_id': session_id}
            })

            # Execute agent
            result = await agent.a_run(query=query, session_id=session_id)

            # Get final summary
            final_summary = progress_printer.tree_builder.get_execution_summary()

            # Extract outline information
            outline_info = progress_printer._get_current_outline_info()

            # Initialize outline_info if empty
            if not outline_info or not outline_info.get('steps'):
                outline_info = {
                    'steps': [],
                    'current_step': 1,
                    'completed_steps': [],
                    'total_steps': 0,
                    'step_descriptions': {},
                    'current_step_progress': "",
                    'outline_raw_data': None,
                    'outline_created': False,
                    'actual_step_completions': []
                }

            # Try to infer outline from execution pattern if not found
            if not outline_info.get('outline_created'):
                outline_info = progress_printer._infer_outline_from_execution_pattern(outline_info)

            # Send final result with summary
            await self._broadcast_to_agent_ui(agent_id, {
                'event': 'chat_response',
                'data': {
                    'response': result,
                    'query': query,
                    'session_id': session_id,
                    'completed_at': asyncio.get_event_loop().time(),
                    'final_summary': {
                        'outline_completed': len(outline_info.get('completed_steps', [])) == outline_info.get(
                            'total_steps', 0),
                        'total_meta_tools': len([e for e in progress_printer.tree_builder.nodes.values()
                                                 for event in e.llm_calls + e.sub_events
                                                 if event.metadata and event.metadata.get('meta_tool_name')]),
                        'total_nodes': final_summary['session_info']['total_nodes'],
                        'execution_time': final_summary['timing']['elapsed'],
                        'total_cost': final_summary['performance_metrics']['total_cost']
                    }
                }
            })

        except Exception as e:
            await self._broadcast_to_agent_ui(agent_id, {
                'event': 'error',
                'data': {'error': str(e), 'query': query}
            })
        finally:
            # Restore original callback
            if hasattr(agent, 'set_progress_callback'):
                agent.set_progress_callback(original_callback)
            elif hasattr(agent, 'progress_callback'):
                agent.progress_callback = original_callback

    # Replace the existing method
    async def _handle_chat_message(self, agent_id: str, agent, conn_id: str, data: dict):
        """Delegate to enhanced handler."""
        await self._handle_chat_message_with_progress_integration(agent_id, agent, conn_id, data)

    # Unified publish and host method
    # toolboxv2/mods/isaa/Tools.py

    async def publish_and_host_agent(
        self,
        agent,
        public_name: str,
        registry_server: str = "ws://localhost:8080/ws/registry/connect",
        description: str | None = None,
        access_level: str = "public"
    ) -> dict[str, Any]:
        """FIXED: Mit Debug-Ausgaben für Troubleshooting."""

        if hasattr(agent, 'name') and not hasattr(agent, 'amd') and hasattr(agent, 'a_run'):
            agent.amd = lambda :None
            agent.amd.name = agent.name

        try:
            # Registry Client initialisieren
            from toolboxv2.mods.registry.client import get_registry_client
            registry_client = get_registry_client(self.app)

            self.app.print(f"Connecting to registry server: {registry_server}")
            await registry_client.connect(registry_server)

            # Progress Callback für Live-Updates einrichten
            callback_success = await self.setup_live_progress_callback(agent, registry_client, f"agent_{agent.amd.name}")
            if not callback_success:
                self.app.print("Warning: Progress callback setup failed")
            else:
                self.app.print("✅ Progress callback setup successful")

            # Agent beim Registry registrieren
            self.app.print(f"Registering agent: {public_name}")
            registration_info = await registry_client.register(
                agent_instance=agent,
                public_name=public_name,
                description=description or f"Agent: {public_name}"
            )

            if not registration_info:
                return {"error": "Registration failed", "success": False}

            self.app.print(f"✅ Agent registration successful: {registration_info.public_agent_id}")

            result = {
                "success": True,
                "agent_name": public_name,
                "public_agent_id": registration_info.public_agent_id,
                "public_api_key": registration_info.public_api_key,
                "public_url": registration_info.public_url,
                "registry_server": registry_server,
                "access_level": access_level,
                "ui_url": registration_info.public_url.replace("/api/registry/run", "/api/registry/ui"),
                "websocket_url": registry_server.replace("/connect", "/ui_connect"),
                "status": "registered"
            }

            return result

        except Exception as e:
            self.app.print(f"Failed to publish agent: {e}")
            return {"error": str(e), "success": False}

    # toolboxv2/mods/isaa/Tools.py

    async def setup_live_progress_callback(self, agent, registry_client, agent_id: str = None):
        """Enhanced setup for live progress callback with proper error handling."""

        if not registry_client:
            self.app.print("Warning: No registry client provided for progress updates")
            return False

        if not registry_client.is_connected:
            self.app.print("Warning: Registry client is not connected")
            return False

        progress_tracker = EnhancedProgressTracker()

        # Generate agent ID if not provided
        if not agent_id:
            agent_id = getattr(agent, 'name', f'agent_{id(agent)}')

        async def enhanced_live_progress_callback(event: ProgressEvent):
            """Enhanced progress callback with comprehensive data extraction."""
            try:
                # Validate event
                if not event:
                    self.app.print("Warning: Received null progress event")
                    return

                # Debug output for local development
                event_type = getattr(event, 'event_type', 'unknown')
                status = getattr(event, 'status', 'unknown')
                agent_name = getattr(event, 'agent_name', 'Unknown Agent')

                self.app.print(f"📊 Progress Event: {event_type} | {status} | {agent_name}")

                # Extract comprehensive progress data
                progress_data = progress_tracker.extract_progress_data(event)

                # Prepare enhanced progress message
                ui_progress_data = {
                    "agent_id": agent_id,
                    "event_type": event_type,
                    "status": status.value if hasattr(status, 'value') else str(status),
                    "timestamp": getattr(event, 'timestamp', asyncio.get_event_loop().time()),
                    "agent_name": agent_name,
                    "node_name": getattr(event, 'node_name', 'Unknown'),
                    "session_id": getattr(event, 'session_id', None),

                    # Core event metadata
                    "metadata": {
                        **getattr(event, 'metadata', {}),
                        "event_id": getattr(event, 'event_id', f"evt_{asyncio.get_event_loop().time()}"),
                        "sequence_number": getattr(event, 'sequence_number', 0),
                        "parent_event_id": getattr(event, 'parent_event_id', None)
                    },

                    # Detailed progress data for UI panels
                    "progress_data": progress_data,

                    # UI-specific flags for selective updates
                    "ui_flags": {
                        "should_update_outline": bool(progress_data.get('outline')),
                        "should_update_activity": bool(progress_data.get('activity')),
                        "should_update_meta_tools": bool(progress_data.get('meta_tool')),
                        "should_update_system": bool(progress_data.get('system')),
                        "should_update_graph": bool(progress_data.get('graph')),
                        "is_error": event_type.lower() in ['error', 'exception', 'failed'],
                        "is_completion": event_type.lower() in ['complete', 'finished', 'success'],
                        "requires_user_input": getattr(event, 'requires_user_input', False)
                    },

                    # Performance metrics
                    "performance": {
                        "execution_time": getattr(event, 'execution_time', None),
                        "memory_delta": getattr(event, 'memory_delta', None),
                        "tokens_used": getattr(event, 'tokens_used', None),
                        "api_calls_made": getattr(event, 'api_calls_made', None)
                    }
                }

                # Send live update to registry server
                await registry_client.send_ui_progress(ui_progress_data)

                # Also send agent status update if this is a significant event
                if event_type in ['started', 'completed', 'error', 'paused', 'resumed']:
                    agent_status = 'processing'
                    if event_type == 'completed':
                        agent_status = 'idle'
                    elif event_type == 'error':
                        agent_status = 'error'
                    elif event_type == 'paused':
                        agent_status = 'paused'

                    await registry_client.send_agent_status(
                        agent_id=agent_id,
                        status=agent_status,
                        details={
                            "last_event": event_type,
                            "last_update": ui_progress_data["timestamp"],
                            "current_node": progress_data.get('graph', {}).get('current_node', 'Unknown')
                        }
                    )

                # Log successful progress update
                self.app.print(f"✅ Sent progress update: {event_type} -> Registry Server")

            except Exception as e:
                self.app.print(f"❌ Progress callback error: {e}")
                # Send error notification to UI
                try:
                    await registry_client.send_ui_progress({
                        "agent_id": agent_id,
                        "event_type": "progress_callback_error",
                        "status": "error",
                        "timestamp": asyncio.get_event_loop().time(),
                        "agent_name": getattr(agent, 'name', 'Unknown'),
                        "metadata": {"error": str(e)},
                        "ui_flags": {"is_error": True}
                    })
                except Exception as nested_error:
                    self.app.print(f"Failed to send error notification: {nested_error}")

        # Set up progress callback with enhanced error handling
        callback_set = False

        if hasattr(agent, 'set_progress_callback'):
            try:
                self.app.print(f"🔧 Setting progress callback via set_progress_callback for agent: {agent_id}")
                agent.set_progress_callback(enhanced_live_progress_callback)
                callback_set = True
            except Exception as e:
                self.app.print(f"Failed to set progress callback via set_progress_callback: {e}")

        if not callback_set and hasattr(agent, 'progress_callback'):
            try:
                self.app.print(f"🔧 Setting progress callback via direct assignment for agent: {agent_id}")
                agent.progress_callback = enhanced_live_progress_callback
                callback_set = True
            except Exception as e:
                self.app.print(f"Failed to set progress callback via direct assignment: {e}")

        if not callback_set:
            self.app.print(f"⚠️ Warning: Agent {agent_id} doesn't support progress callbacks")
            return False

        # Send initial agent status
        try:
            await registry_client.send_agent_status(
                agent_id=agent_id,
                status='online',
                details={
                    "progress_callback_enabled": True,
                    "callback_setup_time": asyncio.get_event_loop().time(),
                    "agent_type": type(agent).__name__
                }
            )
            self.app.print(f"✅ Progress callback successfully set up for agent: {agent_id}")
        except Exception as e:
            self.app.print(f"Failed to send initial agent status: {e}")

        return True


    async def _setup_builtin_server_hosting(self, agent_id: str, agent, host, port) -> dict[str, str]:
        """Setup agent hosting using toolbox built-in server with enhanced WebSocket support."""

        # Register WebSocket handlers for this agent
        @self.app.tb(mod_name="agent_ui", websocket_handler="connect")
        def register_agent_ws_handlers(_):
            return {
                "on_connect": self._create_agent_ws_connect_handler(agent_id),
                "on_message": self._create_agent_ws_message_handler(agent_id, agent),
                "on_disconnect": self._create_agent_ws_disconnect_handler(agent_id),
            }

        # Register UI endpoint - now uses enhanced UI
        @self.app.tb(mod_name="agent_ui", api=True, version="1", api_methods=['GET'])
        async def ui():
            return Result.html(
                self._get_enhanced_agent_ui_html(agent_id), row=True
            )

        # Register API endpoint for direct agent interaction
        @self.app.tb(mod_name="agent_ui", api=True, version="1", request_as_kwarg=True, api_methods=['POST'])
        async def run_agent(request: RequestData):
            return await self._handle_direct_agent_run(agent_id, agent, request)

        # Register additional API endpoints for enhanced features
        @self.app.tb(mod_name="agent_ui", api=True, version="1", request_as_kwarg=True, api_methods=['POST'])
        async def reset_context(request: RequestData):
            return await self._handle_api_reset_context(agent_id, agent, request)

        @self.app.tb(mod_name="agent_ui", api=True, version="1", request_as_kwarg=True, api_methods=['GET'])
        async def status(request: RequestData):
            return await self._handle_api_get_status(agent_id, agent, request)

        # WebSocket endpoint URL
        uri = f"{host}:{port}" if port else f"{host}"
        ws_url = f"ws://{uri}/ws/agent_ui/connect"
        ui_url = f"http://{uri}/api/agent_ui/ui"
        api_url = f"http://{uri}/api/agent_ui/run_agent"

        return {
            'ui_url': ui_url,
            'ws_url': ws_url,
            'api_url': api_url,
            'reset_url': f"http://localhost:{self.app.args_sto.port}/api/agent_ui/reset_context",
            'status_url': f"http://localhost:{self.app.args_sto.port}/api/agent_ui/status",
            'server_type': 'builtin',
            'status': 'running'
        }

    async def _setup_standalone_server_hosting(self, agent_id: str, agent, host: str, port: int) -> dict[str, str]:
        """Setup agent hosting using standalone Python HTTP server with enhanced UI support."""

        if not hasattr(self, '_standalone_servers'):
            self._standalone_servers = {}

        if port in self._standalone_servers:
            self.app.print(f"Port {port} is already in use by another agent")
            return {'status': 'error', 'error': f'Port {port} already in use'}

        # Store server info for the handler
        server_info = {
            'agent_id': agent_id,
            'server_type': 'standalone',
            'api_paths': {
                'ui': '/ui',
                'status': '/api/status',
                'run': '/api/run',
                'reset': '/api/reset'
            }
        }

        # Create handler factory with agent reference and server info
        def handler_factory(*args, **kwargs):
            handler = EnhancedAgentRequestHandler(self, agent_id, agent, *args, **kwargs)
            handler.server_info = server_info
            return handler

        # Start HTTP server in separate thread
        def run_server():
            try:
                httpd = HTTPServer((host, port), handler_factory)
                self._standalone_servers[port] = {
                    'server': httpd,
                    'agent_id': agent_id,
                    'thread': threading.current_thread(),
                    'server_info': server_info
                }

                self.app.print(f"Enhanced standalone server for agent '{agent_id}' running on http://{host}:{port}")
                self.app.print(f"  UI: http://{host}:{port}/ui")
                self.app.print(f"  API: http://{host}:{port}/api/run")
                self.app.print(f"  Status: http://{host}:{port}/api/status")

                httpd.serve_forever()

            except Exception as e:
                self.app.print(f"Standalone server failed: {e}")
            finally:
                if port in self._standalone_servers:
                    del self._standalone_servers[port]

        # Start server in daemon thread
        server_thread = threading.Thread(target=run_server, daemon=True)
        server_thread.start()

        # Wait a moment to ensure server starts
        await asyncio.sleep(0.5)

        return {
            'server_type': 'standalone',
            'local_url': f"http://{host}:{port}",
            'ui_url': f"http://{host}:{port}/ui",
            'api_url': f"http://{host}:{port}/api/run",
            'reset_url': f"http://{host}:{port}/api/reset",
            'status_url': f"http://{host}:{port}/api/status",
            'status': 'running',
            'port': port
        }

    async def _handle_direct_agent_run(self, agent_id: str, agent, request_data) -> Result:
        """Handle direct agent API calls with enhanced progress tracking."""

        try:
            # Parse request body
            body = request_data.body if hasattr(request_data, 'body') else {}

            if not isinstance(body, dict):
                return Result.default_user_error("Request body must be JSON object", exec_code=400)

            query = body.get('query', '')
            session_id = body.get('session_id', f'api_{secrets.token_hex(8)}')
            kwargs = body.get('kwargs', {})
            include_progress = body.get('include_progress', True)

            if not query:
                return Result.default_user_error("Missing 'query' field in request body", exec_code=400)

            # Enhanced progress tracking for API
            progress_events = []
            enhanced_progress = {}

            async def enhanced_api_progress_callback(event):
                if include_progress:
                    progress_tracker = EnhancedProgressTracker()
                    progress_data = progress_tracker.extract_progress_data(event)

                    progress_events.append({
                        'timestamp': event.timestamp,
                        'event_type': event.event_type,
                        'status': event.status.value if event.status else 'unknown',
                        'agent_name': event.agent_name,
                        'metadata': event.metadata
                    })

                    # Store enhanced progress data
                    enhanced_progress.update(progress_data)

            # Set progress callback
            original_callback = getattr(agent, 'progress_callback', None)

            try:
                if hasattr(agent, 'set_progress_callback'):
                    agent.set_progress_callback(enhanced_api_progress_callback)
                elif hasattr(agent, 'progress_callback'):
                    agent.progress_callback = enhanced_api_progress_callback

                # Execute agent
                result = await agent.a_run(query=query, session_id=session_id, **kwargs)

                # Return enhanced structured response
                response_data = {
                    'success': True,
                    'result': result,
                    'session_id': session_id,
                    'agent_id': agent_id,
                    'execution_time': time.time()
                }

                if include_progress:
                    response_data.update({
                        'progress_events': progress_events,
                        'enhanced_progress': enhanced_progress,
                        'outline_info': enhanced_progress.get('outline', {}),
                        'system_info': enhanced_progress.get('system', {}),
                        'meta_tools_used': enhanced_progress.get('meta_tools', [])
                    })

                return Result.json(data=response_data)

            except Exception as e:
                self.app.print(f"Agent execution error: {e}")
                return Result.default_internal_error(
                    info=f"Agent execution failed: {str(e)}",
                    exec_code=500
                )
            finally:
                # Restore original callback
                if hasattr(agent, 'set_progress_callback'):
                    agent.set_progress_callback(original_callback)
                elif hasattr(agent, 'progress_callback'):
                    agent.progress_callback = original_callback

        except Exception as e:
            self.app.print(f"Direct agent run error: {e}")
            return Result.default_internal_error(
                info=f"Request processing failed: {str(e)}",
                exec_code=500
            )

    async def _handle_api_reset_context(self, agent_id: str, agent, request_data) -> Result:
        """Handle API context reset requests."""
        try:
            if hasattr(agent, 'clear_context'):
                agent.clear_context()
                message = "Context reset successfully"
                success = True
            elif hasattr(agent, 'reset'):
                agent.reset()
                message = "Agent reset successfully"
                success = True
            else:
                message = "Agent does not support context reset"
                success = False

            return Result.json(data={
                'success': success,
                'message': message,
                'agent_id': agent_id,
                'timestamp': time.time()
            })

        except Exception as e:
            return Result.default_internal_error(
                info=f"Context reset failed: {str(e)}",
                exec_code=500
            )

    async def _handle_api_get_status(self, agent_id: str, agent, request_data) -> Result:
        """Handle API status requests."""
        try:
            # Collect comprehensive agent status
            status_info = {
                'agent_id': agent_id,
                'agent_name': getattr(agent, 'name', 'Unknown'),
                'agent_type': agent.__class__.__name__,
                'status': 'active',
                'timestamp': time.time(),
                'server_type': 'api'
            }

            # Add agent-specific status
            if hasattr(agent, 'status'):
                try:
                    agent_status = agent.status()
                    if isinstance(agent_status, dict):
                        status_info['agent_status'] = agent_status
                except:
                    pass

            # Add hosted agent info
            if hasattr(self, '_hosted_agents') and agent_id in self._hosted_agents:
                hosted_info = self._hosted_agents[agent_id]
                status_info.update({
                    'host': hosted_info.get('host'),
                    'port': hosted_info.get('port'),
                    'access': hosted_info.get('access'),
                    'public_name': hosted_info.get('public_name'),
                    'description': hosted_info.get('description')
                })

            # Add connection info
            connection_count = 0
            if hasattr(self, '_agent_connections') and agent_id in self._agent_connections:
                connection_count = len(self._agent_connections[agent_id])

            status_info['active_connections'] = connection_count

            return Result.json(data=status_info)

        except Exception as e:
            return Result.default_internal_error(
                info=f"Status retrieval failed: {str(e)}",
                exec_code=500
            )

cleanup_tools_interfaces() async

Cleanup all ToolsInterface instances.

Source code in toolboxv2/mods/isaa/module.py

async def cleanup_tools_interfaces(self):
    """
    Cleanup all ToolsInterface instances.
    """
    if not hasattr(self, 'tools_interfaces'):
        return

    async def cleanup_async():
        for name, tools_interface in self.tools_interfaces.items():
            if tools_interface:
                try:
                    await tools_interface.__aexit__(None, None, None)
                except Exception as e:
                    self.print(f"Error cleaning up ToolsInterface for {name}: {e}")

    # Run cleanup
    try:
        await cleanup_async()
        self.tools_interfaces.clear()
        self.print("Cleaned up all ToolsInterface instances")
    except Exception as e:
        self.print(f"Error during ToolsInterface cleanup: {e}")

configure_tools_interface(agent_name, **kwargs) async

Configure the ToolsInterface for a specific agent.

Parameters:

Name	Type	Description	Default
agent_name	str	Name of the agent	required
**kwargs		Configuration parameters	{}

Returns:

Type	Description
bool	True if successful, False otherwise

Source code in toolboxv2/mods/isaa/module.py

async def configure_tools_interface(self, agent_name: str, **kwargs) -> bool:
    """
    Configure the ToolsInterface for a specific agent.

    Args:
        agent_name: Name of the agent
        **kwargs: Configuration parameters

    Returns:
        True if successful, False otherwise
    """
    tools_interface = self.get_tools_interface(agent_name)
    if not tools_interface:
        self.print(f"No ToolsInterface found for agent {agent_name}")
        return False

    try:
        # Configure based on provided parameters
        if 'base_directory' in kwargs:
            await tools_interface.set_base_directory(kwargs['base_directory'])

        if 'current_file' in kwargs:
            await tools_interface.set_current_file(kwargs['current_file'])

        if 'variables' in kwargs:
            tools_interface.ipython.user_ns.update(kwargs['variables'])

        self.print(f"Configured ToolsInterface for agent {agent_name}")
        return True

    except Exception as e:
        self.print(f"Failed to configure ToolsInterface for {agent_name}: {e}")
        return False

get_tools_interface(agent_name='self')

Get the ToolsInterface instance for a specific agent.

Parameters:

Name	Type	Description	Default
agent_name	str	Name of the agent	'self'

Returns:

Type	Description
ToolsInterface | None	ToolsInterface instance or None if not found

Source code in toolboxv2/mods/isaa/module.py

def get_tools_interface(self, agent_name: str = "self") -> ToolsInterface | None:
    """
    Get the ToolsInterface instance for a specific agent.

    Args:
        agent_name: Name of the agent

    Returns:
        ToolsInterface instance or None if not found
    """
    if not hasattr(self, 'tools_interfaces'):
        return None

    return self.tools_interfaces.get(agent_name)

host_agent_ui(agent, host='0.0.0.0', port=None, access='local', registry_server=None, public_name=None, description=None, use_builtin_server=None) async

Unified agent hosting with WebSocket-enabled UI and optional registry publishing.

Parameters:

Name	Type	Description	Default
agent		Agent or Chain instance to host	required
host	str	Host address (default: 0.0.0.0 for remote access)	'0.0.0.0'
port	int | None	Port number (auto-assigned if None)	None
access	str	'local', 'remote', or 'registry'	'local'
registry_server	str | None	Registry server URL for publishing (e.g., "ws://localhost:8080/ws/registry/connect")	None
public_name	str | None	Public name for registry publishing	None
description	str | None	Description for registry publishing	None
use_builtin_server	bool	Use toolbox built-in server vs standalone Python server	None

Returns:

Type	Description
dict[str, str]	Dictionary with access URLs and configuration

Source code in toolboxv2/mods/isaa/module.py

async def host_agent_ui(
    self,
    agent,
    host: str = "0.0.0.0",
    port: int | None = None,
    access: str = 'local',
    registry_server: str | None = None,
    public_name: str | None = None,
    description: str | None = None,
    use_builtin_server: bool = None
) -> dict[str, str]:
    """
    Unified agent hosting with WebSocket-enabled UI and optional registry publishing.

    Args:
        agent: Agent or Chain instance to host
        host: Host address (default: 0.0.0.0 for remote access)
        port: Port number (auto-assigned if None)
        access: 'local', 'remote', or 'registry'
        registry_server: Registry server URL for publishing (e.g., "ws://localhost:8080/ws/registry/connect")
        public_name: Public name for registry publishing
        description: Description for registry publishing
        use_builtin_server: Use toolbox built-in server vs standalone Python server

    Returns:
        Dictionary with access URLs and configuration
    """
    use_builtin_server = use_builtin_server or self.app.is_server
    if not hasattr(self, '_hosted_agents'):
        self._hosted_agents = {}

    agent_id = f"agent_{secrets.token_urlsafe(8)}"

    # Generate unique port if not specified
    if not port:
        port = 8765 + len(self._hosted_agents)

    # Store agent reference
    self._hosted_agents[agent_id] = {
        'agent': agent,
        'port': port,
        'host': host,
        'access': access,
        'public_name': public_name or f"Agent_{agent_id}",
        'description': description
    }

    result = {
        'agent_id': agent_id,
        'local_url': f"http://{host}:{port}",
        'status': 'starting'
    }

    if use_builtin_server:
        # Use toolbox built-in server
        result.update(await self._setup_builtin_server_hosting(agent_id, agent, host, port))
    else:
        # Use standalone Python server
        result.update(await self._setup_standalone_server_hosting(agent_id, agent, host, port))

    # Handle registry publishing if requested
    if access in ['remote', 'registry'] and registry_server:
        if not public_name:
            raise ValueError("public_name required for registry publishing")

        registry_result = await self._publish_to_registry(
            agent=agent,
            public_name=public_name,
            registry_server=registry_server,
            description=description,
            agent_id=agent_id
        )
        result.update(registry_result)

    self.app.print(f"🚀 Agent '{result.get('public_name', agent_id)}' hosted successfully!")
    self.app.print(f"   Local UI: {result['local_url']}")
    if 'public_url' in result:
        self.app.print(f"   Public URL: {result['public_url']}")
        self.app.print(f"   API Key: {result.get('api_key', 'N/A')}")

    return result

init_from_augment(augment, agent_name='self') async

Initialize from augmented data using new builder system

Source code in toolboxv2/mods/isaa/module.py

async def init_from_augment(self, augment, agent_name: str = 'self'):
    """Initialize from augmented data using new builder system"""

    # Handle agent_name parameter
    if isinstance(agent_name, str):
        pass  # Use string name
    elif hasattr(agent_name, 'config'):  # FlowAgentBuilder
        agent_name = agent_name.config.name
    else:
        raise ValueError(f"Invalid agent_name type: {type(agent_name)}")

    a_keys = augment.keys()

    # Load agent configurations
    if "Agents" in a_keys:
        agents_configs_dict = augment['Agents']
        self.deserialize_all(agents_configs_dict)
        self.print("Agent configurations loaded.")

    # Tools are now handled by the builder system during agent creation
    if "tools" in a_keys:
        self.print("Tool configurations noted - will be applied during agent building")

list_hosted_agents() async

List all currently hosted agents.

Source code in toolboxv2/mods/isaa/module.py

async def list_hosted_agents(self) -> dict[str, Any]:
    """List all currently hosted agents."""

    hosted_info = {
        'builtin_agents': {},
        'standalone_agents': {},
        'total_count': 0
    }

    # Built-in server agents
    if hasattr(self, '_hosted_agents'):
        for agent_id, info in self._hosted_agents.items():
            hosted_info['builtin_agents'][agent_id] = {
                'public_name': info.get('public_name'),
                'host': info.get('host'),
                'port': info.get('port'),
                'access': info.get('access'),
                'description': info.get('description')
            }

    # Standalone server agents
    if hasattr(self, '_standalone_servers'):
        for port, info in self._standalone_servers.items():
            hosted_info['standalone_agents'][info['agent_id']] = {
                'port': port,
                'thread_alive': info['thread'].is_alive(),
                'server_type': 'standalone'
            }

    hosted_info['total_count'] = len(hosted_info['builtin_agents']) + len(hosted_info['standalone_agents'])

    return hosted_info

publish_and_host_agent(agent, public_name, registry_server='ws://localhost:8080/ws/registry/connect', description=None, access_level='public') async

FIXED: Mit Debug-Ausgaben für Troubleshooting.

Source code in toolboxv2/mods/isaa/module.py

async def publish_and_host_agent(
    self,
    agent,
    public_name: str,
    registry_server: str = "ws://localhost:8080/ws/registry/connect",
    description: str | None = None,
    access_level: str = "public"
) -> dict[str, Any]:
    """FIXED: Mit Debug-Ausgaben für Troubleshooting."""

    if hasattr(agent, 'name') and not hasattr(agent, 'amd') and hasattr(agent, 'a_run'):
        agent.amd = lambda :None
        agent.amd.name = agent.name

    try:
        # Registry Client initialisieren
        from toolboxv2.mods.registry.client import get_registry_client
        registry_client = get_registry_client(self.app)

        self.app.print(f"Connecting to registry server: {registry_server}")
        await registry_client.connect(registry_server)

        # Progress Callback für Live-Updates einrichten
        callback_success = await self.setup_live_progress_callback(agent, registry_client, f"agent_{agent.amd.name}")
        if not callback_success:
            self.app.print("Warning: Progress callback setup failed")
        else:
            self.app.print("✅ Progress callback setup successful")

        # Agent beim Registry registrieren
        self.app.print(f"Registering agent: {public_name}")
        registration_info = await registry_client.register(
            agent_instance=agent,
            public_name=public_name,
            description=description or f"Agent: {public_name}"
        )

        if not registration_info:
            return {"error": "Registration failed", "success": False}

        self.app.print(f"✅ Agent registration successful: {registration_info.public_agent_id}")

        result = {
            "success": True,
            "agent_name": public_name,
            "public_agent_id": registration_info.public_agent_id,
            "public_api_key": registration_info.public_api_key,
            "public_url": registration_info.public_url,
            "registry_server": registry_server,
            "access_level": access_level,
            "ui_url": registration_info.public_url.replace("/api/registry/run", "/api/registry/ui"),
            "websocket_url": registry_server.replace("/connect", "/ui_connect"),
            "status": "registered"
        }

        return result

    except Exception as e:
        self.app.print(f"Failed to publish agent: {e}")
        return {"error": str(e), "success": False}

setup_live_progress_callback(agent, registry_client, agent_id=None) async

Enhanced setup for live progress callback with proper error handling.

Source code in toolboxv2/mods/isaa/module.py

async def setup_live_progress_callback(self, agent, registry_client, agent_id: str = None):
    """Enhanced setup for live progress callback with proper error handling."""

    if not registry_client:
        self.app.print("Warning: No registry client provided for progress updates")
        return False

    if not registry_client.is_connected:
        self.app.print("Warning: Registry client is not connected")
        return False

    progress_tracker = EnhancedProgressTracker()

    # Generate agent ID if not provided
    if not agent_id:
        agent_id = getattr(agent, 'name', f'agent_{id(agent)}')

    async def enhanced_live_progress_callback(event: ProgressEvent):
        """Enhanced progress callback with comprehensive data extraction."""
        try:
            # Validate event
            if not event:
                self.app.print("Warning: Received null progress event")
                return

            # Debug output for local development
            event_type = getattr(event, 'event_type', 'unknown')
            status = getattr(event, 'status', 'unknown')
            agent_name = getattr(event, 'agent_name', 'Unknown Agent')

            self.app.print(f"📊 Progress Event: {event_type} | {status} | {agent_name}")

            # Extract comprehensive progress data
            progress_data = progress_tracker.extract_progress_data(event)

            # Prepare enhanced progress message
            ui_progress_data = {
                "agent_id": agent_id,
                "event_type": event_type,
                "status": status.value if hasattr(status, 'value') else str(status),
                "timestamp": getattr(event, 'timestamp', asyncio.get_event_loop().time()),
                "agent_name": agent_name,
                "node_name": getattr(event, 'node_name', 'Unknown'),
                "session_id": getattr(event, 'session_id', None),

                # Core event metadata
                "metadata": {
                    **getattr(event, 'metadata', {}),
                    "event_id": getattr(event, 'event_id', f"evt_{asyncio.get_event_loop().time()}"),
                    "sequence_number": getattr(event, 'sequence_number', 0),
                    "parent_event_id": getattr(event, 'parent_event_id', None)
                },

                # Detailed progress data for UI panels
                "progress_data": progress_data,

                # UI-specific flags for selective updates
                "ui_flags": {
                    "should_update_outline": bool(progress_data.get('outline')),
                    "should_update_activity": bool(progress_data.get('activity')),
                    "should_update_meta_tools": bool(progress_data.get('meta_tool')),
                    "should_update_system": bool(progress_data.get('system')),
                    "should_update_graph": bool(progress_data.get('graph')),
                    "is_error": event_type.lower() in ['error', 'exception', 'failed'],
                    "is_completion": event_type.lower() in ['complete', 'finished', 'success'],
                    "requires_user_input": getattr(event, 'requires_user_input', False)
                },

                # Performance metrics
                "performance": {
                    "execution_time": getattr(event, 'execution_time', None),
                    "memory_delta": getattr(event, 'memory_delta', None),
                    "tokens_used": getattr(event, 'tokens_used', None),
                    "api_calls_made": getattr(event, 'api_calls_made', None)
                }
            }

            # Send live update to registry server
            await registry_client.send_ui_progress(ui_progress_data)

            # Also send agent status update if this is a significant event
            if event_type in ['started', 'completed', 'error', 'paused', 'resumed']:
                agent_status = 'processing'
                if event_type == 'completed':
                    agent_status = 'idle'
                elif event_type == 'error':
                    agent_status = 'error'
                elif event_type == 'paused':
                    agent_status = 'paused'

                await registry_client.send_agent_status(
                    agent_id=agent_id,
                    status=agent_status,
                    details={
                        "last_event": event_type,
                        "last_update": ui_progress_data["timestamp"],
                        "current_node": progress_data.get('graph', {}).get('current_node', 'Unknown')
                    }
                )

            # Log successful progress update
            self.app.print(f"✅ Sent progress update: {event_type} -> Registry Server")

        except Exception as e:
            self.app.print(f"❌ Progress callback error: {e}")
            # Send error notification to UI
            try:
                await registry_client.send_ui_progress({
                    "agent_id": agent_id,
                    "event_type": "progress_callback_error",
                    "status": "error",
                    "timestamp": asyncio.get_event_loop().time(),
                    "agent_name": getattr(agent, 'name', 'Unknown'),
                    "metadata": {"error": str(e)},
                    "ui_flags": {"is_error": True}
                })
            except Exception as nested_error:
                self.app.print(f"Failed to send error notification: {nested_error}")

    # Set up progress callback with enhanced error handling
    callback_set = False

    if hasattr(agent, 'set_progress_callback'):
        try:
            self.app.print(f"🔧 Setting progress callback via set_progress_callback for agent: {agent_id}")
            agent.set_progress_callback(enhanced_live_progress_callback)
            callback_set = True
        except Exception as e:
            self.app.print(f"Failed to set progress callback via set_progress_callback: {e}")

    if not callback_set and hasattr(agent, 'progress_callback'):
        try:
            self.app.print(f"🔧 Setting progress callback via direct assignment for agent: {agent_id}")
            agent.progress_callback = enhanced_live_progress_callback
            callback_set = True
        except Exception as e:
            self.app.print(f"Failed to set progress callback via direct assignment: {e}")

    if not callback_set:
        self.app.print(f"⚠️ Warning: Agent {agent_id} doesn't support progress callbacks")
        return False

    # Send initial agent status
    try:
        await registry_client.send_agent_status(
            agent_id=agent_id,
            status='online',
            details={
                "progress_callback_enabled": True,
                "callback_setup_time": asyncio.get_event_loop().time(),
                "agent_type": type(agent).__name__
            }
        )
        self.app.print(f"✅ Progress callback successfully set up for agent: {agent_id}")
    except Exception as e:
        self.app.print(f"Failed to send initial agent status: {e}")

    return True

stop_hosted_agent(agent_id=None, port=None) async

Stop a hosted agent by agent_id or port.

Source code in toolboxv2/mods/isaa/module.py

async def stop_hosted_agent(self, agent_id: str = None, port: int = None):
    """Stop a hosted agent by agent_id or port."""

    if not hasattr(self, '_hosted_agents') and not hasattr(self, '_standalone_servers'):
        self.app.print("No hosted agents found")
        return False

    # Stop by agent_id
    if agent_id:
        if hasattr(self, '_hosted_agents') and agent_id in self._hosted_agents:
            agent_info = self._hosted_agents[agent_id]
            agent_port = agent_info.get('port')

            # Stop standalone server if exists
            if hasattr(self, '_standalone_servers') and agent_port in self._standalone_servers:
                server_info = self._standalone_servers[agent_port]
                try:
                    server_info['server'].shutdown()
                    self.app.print(f"Stopped standalone server for agent {agent_id}")
                except:
                    pass

            # Clean up hosted agent info
            del self._hosted_agents[agent_id]
            self.app.print(f"Stopped hosted agent {agent_id}")
            return True

    # Stop by port
    if port:
        if hasattr(self, '_standalone_servers') and port in self._standalone_servers:
            server_info = self._standalone_servers[port]
            try:
                server_info['server'].shutdown()
                self.app.print(f"Stopped server on port {port}")
                return True
            except Exception as e:
                self.app.print(f"Failed to stop server on port {port}: {e}")
                return False

    self.app.print("Agent or port not found")
    return False

ui

get_agent_ui_html()

Produktionsfertige UI mit Live-Progress-Tracking.

Source code in toolboxv2/mods/isaa/ui.py

def get_agent_ui_html() -> str:
    """Produktionsfertige UI mit Live-Progress-Tracking."""

    return """<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>Agent Registry - Live Interface</title>
    <script src="https://cdn.jsdelivr.net/npm/marked/marked.min.js"></script>
    <style>
        /* Modernes Dark Theme UI */
        :root {
            --bg-primary: #0d1117;
            --bg-secondary: #161b22;
            --bg-tertiary: #21262d;
            --text-primary: #f0f6fc;
            --text-secondary: #8b949e;
            --text-muted: #6e7681;
            --accent-blue: #58a6ff;
            --accent-green: #3fb950;
            --accent-red: #f85149;
            --accent-orange: #d29922;
            --accent-purple: #a5a5f5;
            --accent-cyan: #39d0d8;
            --border-color: #30363d;
            --shadow: 0 2px 8px rgba(0, 0, 0, 0.3);

            --sidebar-width: 300px;
            --progress-width: 660px;
            --sidebar-collapsed: 60px;
            --progress-collapsed: 60px;
        }
        /* Enhanced Progress Panel Styles */
        .progress-section {
            margin-bottom: 16px;
        }

        /* ADD to existing CSS */
        .event-status-badge {
            padding: 2px 6px;
            border-radius: 3px;
            font-size: 10px;
            font-weight: 500;
        }

        .event-status-badge.completed {
            background: var(--accent-green);
            color: white;
        }

        .event-status-badge.running {
            background: var(--accent-orange);
            color: white;
        }

        .event-status-badge.failed, .event-status-badge.error {
            background: var(--accent-red);
            color: white;
        }

        .event-status-badge.starting {
            background: var(--accent-cyan);
            color: white;
        }

        .progress-item.expandable[data-event-id*="tool_call"] {
            border-left-color: var(--accent-orange);
        }

        .progress-item.expandable[data-event-id*="llm_call"] {
            border-left-color: var(--accent-purple);
        }

        .progress-item.expandable[data-event-id*="meta_tool"] {
            border-left-color: var(--accent-cyan);
        }

        .progress-item.expandable[data-event-id*="error"] {
            border-left-color: var(--accent-red);
            background: rgba(248, 81, 73, 0.02);
        }

        .section-title.expandable-section {
            cursor: pointer;
            display: flex;
            justify-content: space-between;
            align-items: center;
            padding: 8px 12px;
            background: var(--bg-primary);
            border: 1px solid var(--border-color);
            border-radius: 6px;
            transition: all 0.2s;
        }

        .section-title.expandable-section:hover {
            background: var(--bg-tertiary);
        }

        .section-toggle {
            transition: transform 0.2s;
            font-size: 12px;
        }

        .section-content {
            max-height: 0;
            overflow: hidden;
            transition: max-height 0.3s ease-out;
            background: var(--bg-primary);
            border-radius: 0 0 6px 6px;
        }

        .section-content.expanded {
            max-height: 900px;
            padding: 12px;
            border: 1px solid var(--border-color);
            border-top: none;
            overflow-y: auto;
        }

        .no-data {
            color: var(--text-muted);
            font-size: 12px;
            text-align: center;
            padding: 12px;
            font-style: italic;
        }

        /* Expandable Progress Items */
        .progress-item.expandable {
            cursor: pointer;
            transition: all 0.2s;
            margin-bottom: 8px;
        }

        .progress-item.expandable:hover {
            transform: translateY(-1px);
            box-shadow: 0 4px 12px rgba(0, 0, 0, 0.15);
        }

        .progress-item.expandable.expanded {
            border-color: var(--accent-blue);
        }

        .progress-item.expandable.latest {
            border-left: 3px solid var(--accent-green);
            background: rgba(63, 185, 80, 0.05);
        }

        .progress-item-header {
            display: flex;
            align-items: center;
            gap: 8px;
            padding: 8px 12px;
        }

        .progress-meta {
            margin-left: auto;
            display: flex;
            align-items: center;
            gap: 8px;
        }

        .expand-indicator {
            transition: transform 0.2s;
            font-size: 12px;
            color: var(--text-muted);
        }

        .progress-item.expanded .expand-indicator {
            transform: rotate(180deg);
        }

        .progress-summary {
            padding: 0 12px 8px 36px;
            font-size: 11px;
            color: var(--text-secondary);
        }

        .progress-item-expanded {
            max-height: 0;
            overflow: hidden;
            transition: max-height 0.3s ease-out;
            background: var(--bg-secondary);
            border-top: 1px solid var(--border-color);
        }

        .progress-item-expanded.active {
            max-height: 400px;
            padding: 12px;
            overflow-y: auto;
        }

        .expanded-section {
            margin-bottom: 12px;
        }

        .expanded-section-title {
            font-size: 12px;
            font-weight: 600;
            color: var(--accent-blue);
            margin-bottom: 6px;
            padding-bottom: 4px;
            border-bottom: 1px solid var(--border-color);
        }

        .event-field {
            display: flex;
            justify-content: space-between;
            align-items: flex-start;
            padding: 4px 0;
            font-size: 11px;
        }

        .event-field-label {
            font-weight: 500;
            color: var(--text-secondary);
            min-width: 80px;
        }

        .event-field-value {
            color: var(--text-primary);
            text-align: right;
            flex: 1;
        }

        .event-field-value.json {
            background: var(--bg-primary);
            border-radius: 4px;
            padding: 6px;
            font-family: monospace;
            font-size: 10px;
            text-align: left;
            white-space: pre-wrap;
            max-height: 100px;
            overflow-y: auto;
        }

        /* ADD to existing CSS */
.thinking-step.outline-step {
    border-color: var(--accent-cyan);
    background: rgba(57, 208, 216, 0.05);
}

.thinking-step.outline-step.completed {
    border-color: var(--accent-green);
    background: rgba(63, 185, 80, 0.05);
}

.thinking-step.outline-step.running {
    border-color: var(--accent-orange);
    background: rgba(210, 153, 34, 0.05);
}

.outline-progress {
    margin: 8px 0;
}

.progress-info {
    display: flex;
    justify-content: space-between;
    align-items: center;
    margin-bottom: 6px;
}

.progress-text {
    font-size: 11px;
    color: var(--text-secondary);
    font-weight: 500;
}

.progress-percentage {
    font-size: 11px;
    color: var(--accent-blue);
    font-weight: 600;
}

.progress-bar-container {
    margin-bottom: 8px;
}

.progress-bar {
    height: 6px;
    background: var(--bg-primary);
    border-radius: 3px;
    overflow: hidden;
    border: 1px solid var(--border-color);
}

.progress-bar-fill {
    height: 100%;
    background: linear-gradient(90deg, var(--accent-cyan), var(--accent-blue));
    transition: width 0.5s ease-out;
    position: relative;
}

.progress-bar-fill::after {
    content: '';
    position: absolute;
    top: 0;
    left: 0;
    right: 0;
    bottom: 0;
    background: linear-gradient(90deg, transparent, rgba(255,255,255,0.2), transparent);
    animation: shimmer 2s infinite;
}

@keyframes shimmer {
    0% { transform: translateX(-100%); }
    100% { transform: translateX(100%); }
}

.step-completed {
    color: var(--accent-green);
    font-size: 10px;
    text-align: center;
    font-weight: 500;
}

.step-working {
    color: var(--accent-orange);
    font-size: 10px;
    text-align: center;
    font-style: italic;
}

.context-info {
    display: flex;
    justify-content: center;
    gap: 12px;
    margin-top: 8px;
    padding-top: 8px;
    border-top: 1px solid var(--border-color);
}

.context-item {
    font-size: 10px;
    color: var(--text-muted);
    background: var(--bg-primary);
    padding: 2px 6px;
    border-radius: 3px;
    border: 1px solid var(--border-color);
}

.thinking-step.plan-created {
    border-color: var(--accent-blue);
    background: rgba(88, 166, 255, 0.05);
}

.plan-details {
    text-align: center;
}

.plan-info {
    display: flex;
    justify-content: center;
    gap: 12px;
    margin-bottom: 8px;
}

.plan-item {
    font-size: 11px;
    color: var(--text-secondary);
    background: var(--bg-primary);
    padding: 4px 8px;
    border-radius: 4px;
    border: 1px solid var(--border-color);
}

.plan-ready, .outline-ready {
    margin-top: 8px;
    color: var(--accent-green);
    font-size: 10px;
    text-align: center;
}

.step-status {
    padding: 2px 6px;
    border-radius: 3px;
    font-size: 10px;
    font-weight: 500;
    margin-left: auto;
}

.step-status.completed {
    background: var(--accent-green);
    color: white;
}

.step-status.running {
    background: var(--accent-orange);
    color: white;
}

.step-status.ready {
    background: var(--accent-blue);
    color: white;
}

        /* Enhanced Chat Integration Styles */
        .thinking-step {
            background: var(--bg-secondary);
            border: 1px solid var(--border-color);
            border-radius: 8px;
            padding: 10px 12px;
            margin: 8px 0;
            font-size: 13px;
            transition: all 0.2s;
        }

        .thinking-step:hover {
            transform: translateY(-1px);
            box-shadow: 0 2px 8px rgba(0, 0, 0, 0.1);
        }

        .thinking-step.reasoning-loop {
            border-color: var(--accent-purple);
            background: rgba(165, 165, 245, 0.05);
        }

        .thinking-step.outline-created {
            border-color: var(--accent-cyan);
            background: rgba(57, 208, 216, 0.05);
        }

        .thinking-step.task-progress.starting {
            border-color: var(--accent-orange);
            background: rgba(210, 153, 34, 0.05);
        }

        .thinking-step.task-progress.completed {
            border-color: var(--accent-green);
            background: rgba(63, 185, 80, 0.05);
        }

        .thinking-step.task-progress.error {
            border-color: var(--accent-red);
            background: rgba(248, 81, 73, 0.05);
        }

        .thinking-step-header {
            display: flex;
            align-items: center;
            gap: 8px;
            font-weight: 600;
            margin-bottom: 8px;
            color: var(--text-primary);
        }

        .step-progress, .step-info, .step-status {
            margin-left: auto;
            font-size: 10px;
            font-weight: normal;
            color: var(--text-muted);
            background: var(--bg-primary);
            padding: 2px 6px;
            border-radius: 3px;
        }

        .priority-badge {
            padding: 2px 6px;
            border-radius: 3px;
            font-size: 10px;
            font-weight: 500;
        }

        .priority-badge.high {
            background: var(--accent-red);
            color: white;
        }

        .priority-badge.normal {
            background: var(--accent-blue);
            color: white;
        }

        .priority-badge.low {
            background: var(--text-muted);
            color: white;
        }

        /* Performance Metrics Grid */
        .metrics-grid {
            display: grid;
            grid-template-columns: repeat(auto-fit, minmax(100px, 1fr));
            gap: 8px;
        }

        .metric-card {
            background: var(--bg-primary);
            border: 1px solid var(--border-color);
            border-radius: 6px;
            padding: 8px;
            text-align: center;
        }

        .metric-label {
            font-size: 10px;
            color: var(--text-muted);
            margin-bottom: 4px;
        }

        .metric-value {
            font-size: 14px;
            font-weight: 600;
            color: var(--accent-blue);
        }

        .reasoning-metrics .metric-grid {
            display: grid;
            grid-template-columns: repeat(3, 1fr);
            gap: 8px;
            margin-bottom: 8px;
        }

        .metric-item {
            display: flex;
            justify-content: space-between;
            align-items: center;
            font-size: 11px;
        }

        .metric-label {
            color: var(--text-muted);
        }

        .metric-value {
            color: var(--text-primary);
            font-weight: 500;
        }

        /* Progress Bar */
        .progress-bar-container {
            margin: 8px 0;
        }

        .progress-bar-info {
            display: flex;
            justify-content: space-between;
            font-size: 10px;
            color: var(--text-muted);
            margin-bottom: 4px;
        }

        .progress-bar {
            height: 4px;
            background: var(--bg-primary);
            border-radius: 2px;
            overflow: hidden;
        }

        .progress-bar-fill {
            height: 100%;
            background: var(--accent-blue);
            transition: width 0.5s ease-out;
        }

        /* Outline Display */
        .outline-steps {
            margin: 8px 0;
        }

        .outline-step {
            display: flex;
            align-items: flex-start;
            gap: 8px;
            margin-bottom: 4px;
            font-size: 11px;
        }

        .step-number {
            color: var(--accent-blue);
            font-weight: 600;
            min-width: 20px;
        }

        .step-text {
            color: var(--text-primary);
            line-height: 1.3;
        }

        .context-metrics {
            display: grid;
            grid-template-columns: repeat(3, 1fr);
            gap: 8px;
            margin-bottom: 8px;
        }

        .context-metric {
            display: flex;
            flex-direction: column;
            align-items: center;
            font-size: 10px;
            padding: 6px;
            background: var(--bg-primary);
            border-radius: 4px;
        }

        .context-label {
            color: var(--text-muted);
            margin-bottom: 2px;
        }

        .context-value {
            color: var(--text-primary);
            font-weight: 600;
        }

        .task-description {
            margin-bottom: 6px;
            font-weight: 500;
        }

        .task-timing {
            font-size: 10px;
            color: var(--accent-green);
        }

        .task-error {
            font-size: 10px;
            color: var(--accent-red);
            background: rgba(248, 81, 73, 0.1);
            padding: 4px;
            border-radius: 3px;
            margin-top: 4px;
        }

        .reasoning-insight {
            margin-top: 8px;
            font-size: 11px;
            color: var(--accent-purple);
            text-align: center;
            font-style: italic;
        }

        .idle-status {
            border-color: var(--accent-green);
            background: rgba(63, 185, 80, 0.02);
        }

        @media (max-width: 1200px) {
            :root {
                --sidebar-width: 250px;
                --progress-width: 580px;
            }
        }

        @media (max-width: 1024px) {
            :root {
                --sidebar-width: 220px;
                --progress-width: 460px;
            }
        }

        .sidebar.collapsed::before {
            content: '📋';
            font-size: 20px;
            display: flex;
            align-items: center;
            justify-content: center;
            padding: 20px 0;
            border-bottom: 1px solid var(--border-color);
        }

        .progress-panel.collapsed::before {
            content: '📊';
            font-size: 20px;
            display: flex;
            align-items: center;
            justify-content: center;
            padding: 20px 0;
            border-bottom: 1px solid var(--border-color);
            writing-mode: vertical-lr;
        }

        .sidebar, .progress-panel {
            transition: all 0.3s cubic-bezier(0.4, 0, 0.2, 1);
        }

        .main-container {
            transition: grid-template-columns 0.3s cubic-bezier(0.4, 0, 0.2, 1);
        }

        * { margin: 0; padding: 0; box-sizing: border-box; }

        body {
            font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Helvetica, Arial, sans-serif;
            background: var(--bg-primary);
            color: var(--text-primary);
            height: 100vh;
            display: flex;
            flex-direction: column;
            overflow: hidden;
        }

        html, body {
            height: 100%;
            overflow: hidden;
        }

        .api-key-modal {
            position: fixed;
            top: 0;
            left: 0;
            right: 0;
            bottom: 0;
            background: rgba(0, 0, 0, 0.8);
            display: flex;
            align-items: center;
            justify-content: center;
            z-index: 1000;
        }

        .api-key-content {
            background: var(--bg-secondary);
            border: 1px solid var(--border-color);
            border-radius: 12px;
            padding: 24px;
            max-width: 500px;
            width: 90%;
            text-align: center;
        }

        .api-key-title {
            font-size: 20px;
            font-weight: 600;
            color: var(--accent-blue);
            margin-bottom: 16px;
        }

        .api-key-description {
            color: var(--text-secondary);
            margin-bottom: 20px;
            line-height: 1.5;
        }

        .api-key-input {
            width: 100%;
            background: var(--bg-primary);
            border: 1px solid var(--border-color);
            border-radius: 8px;
            padding: 12px;
            color: var(--text-primary);
            font-size: 14px;
            margin-bottom: 16px;
        }

        .api-key-button {
            background: var(--accent-blue);
            color: white;
            border: none;
            border-radius: 8px;
            padding: 12px 24px;
            cursor: pointer;
            font-weight: 600;
        }

        /* Updated Header */
        .header {
            background: var(--bg-tertiary);
            padding: 16px 24px;
            border-bottom: 1px solid var(--border-color);
            display: flex;
            align-items: center;
            justify-content: space-between;
            box-shadow: var(--shadow);
            flex-shrink: 0;
        }

        .header-controls {
            display: flex;
            align-items: center;
            gap: 12px;
        }

        .panel-toggle {
            background: var(--bg-secondary);
            border: 1px solid var(--border-color);
            color: var(--text-primary);
            padding: 8px 12px;
            border-radius: 6px;
            cursor: pointer;
            font-size: 12px;
            transition: all 0.2s;
        }

        .panel-toggle:hover {
            background: var(--bg-primary);
        }

        .panel-toggle.active {
            background: var(--accent-blue);
            color: white;
        }

        .logo {
            display: flex;
            align-items: center;
            gap: 12px;
            font-size: 20px;
            font-weight: 700;
            color: var(--accent-blue);
        }

        .connection-status {
            display: flex;
            align-items: center;
            gap: 12px;
        }

        .status-indicator {
            display: flex;
            align-items: center;
            gap: 8px;
            padding: 8px 12px;
            border-radius: 6px;
            font-size: 14px;
            font-weight: 500;
        }

        .status-indicator.connected {
            background: rgba(63, 185, 80, 0.1);
            color: var(--accent-green);
            border: 1px solid var(--accent-green);
        }

        .status-indicator.disconnected {
            background: rgba(248, 81, 73, 0.1);
            color: var(--accent-red);
            border: 1px solid var(--accent-red);
        }

        .status-dot {
            width: 8px;
            height: 8px;
            border-radius: 50%;
            background: currentColor;
            animation: pulse 2s infinite;
        }

        .status-dot.connected { animation: none; }

        @keyframes pulse {
            0%, 100% { opacity: 1; }
            50% { opacity: 0.4; }
        }

        /* FIXED: Better grid layout that properly handles collapsing */
        .main-container {
            display: grid;
            grid-template-areas: "sidebar chat progress";
            grid-template-columns: var(--sidebar-width) 1fr var(--progress-width);
            flex: 1;
            overflow: hidden;
            min-height: 0;
            height: 100%;
        }

        .main-container.sidebar-collapsed {
            grid-template-columns: var(--sidebar-collapsed) 1fr var(--progress-width);
        }

        .main-container.progress-collapsed {
            grid-template-columns: var(--sidebar-width) 1fr var(--progress-collapsed);
        }

        .main-container.both-collapsed {
            grid-template-columns: var(--sidebar-collapsed) 1fr var(--progress-collapsed);
        }

        .sidebar {
            grid-area: sidebar;
            background: var(--bg-secondary);
            border-right: 1px solid var(--border-color);
            display: flex;
            flex-direction: column;
            overflow: hidden;
            height: 100%;
        }

        .sidebar.collapsed .agents-list,
        .sidebar.collapsed .system-info {
            display: none;
        }

        .sidebar.collapsed .sidebar-header {
            padding: 12px 8px;
            justify-content: center;
        }

        .sidebar.collapsed .sidebar-title {
            display: none;
        }

        .sidebar.collapsed .collapse-btn {
            writing-mode: vertical-lr;
            text-orientation: mixed;
        }

        .progress-panel.collapsed .collapse-btn {
            writing-mode: vertical-lr;
            text-orientation: mixed;
            transform: rotate(180deg);
        }

        .sidebar-header {
            padding: 12px 16px;
            background: var(--bg-tertiary);
            border-bottom: 1px solid var(--border-color);
            display: flex;
            align-items: center;
            justify-content: space-between;
            min-height: 48px;
        }

        .sidebar-title {
            font-size: 14px;
            font-weight: 600;
            color: var(--text-secondary);
            text-transform: uppercase;
        }

        .collapse-btn {
            background: none;
            border: none;
            color: var(--text-muted);
            cursor: pointer;
            padding: 4px;
            border-radius: 4px;
            transition: all 0.2s;
        }

        .collapse-btn:hover {
            background: var(--bg-primary);
            color: var(--text-primary);
        }

        /* FIXED: Chat area properly uses grid area and expands */
        .chat-area {
            grid-area: chat;
            display: flex;
            flex-direction: column;
            background: var(--bg-primary);
            min-height: 0;
            height: 100%;
            overflow: hidden;
        }

        /* Updated Progress Panel */
        .progress-panel {
            grid-area: progress;
            background: var(--bg-secondary);
            border-left: 1px solid var(--border-color);
            display: flex;
            flex-direction: column;
            overflow: hidden;
            height: 100%;
        }

        .progress-panel.collapsed .panel-content {
            display: none;
        }

        .progress-panel.collapsed .progress-header {
            padding: 12px 8px;
            justify-content: center;
            writing-mode: vertical-lr;
            text-orientation: mixed;
        }

        .progress-panel.collapsed .progress-header span {
            transform: rotate(180deg);
        }

        .progress-header {
            padding: 12px 16px;
            background: var(--bg-tertiary);
            border-bottom: 1px solid var(--border-color);
            display: flex;
            align-items: center;
            justify-content: space-between;
            font-weight: 600;
            font-size: 14px;
            min-height: 48px;
        }

        /* ADD to existing CSS */
        .progress-item.llm_call {
            border-left: 3px solid var(--accent-purple);
        }

        .progress-item.llm_call.latest {
            border-left: 3px solid var(--accent-purple);
            background: rgba(165, 165, 245, 0.03);
        }

        .progress-item.llm_call .progress-icon {
            color: var(--accent-purple);
        }

        .progress-summary {
            padding: 0 12px 8px 36px;
            font-size: 10px;
            color: var(--text-secondary);
            line-height: 1.3;
        }

        .event-field-value.json {
            background: var(--bg-primary);
            border-radius: 4px;
            padding: 8px;
            font-family: 'Consolas', 'Monaco', monospace;
            font-size: 10px;
            text-align: left;
            white-space: pre-wrap;
            max-height: 300px;
            overflow-y: auto;
            border: 1px solid var(--border-color);
            word-break: break-all;
        }

        .expanded-section {
            margin-bottom: 12px;
            border-bottom: 1px solid rgba(48, 54, 61, 0.3);
            padding-bottom: 8px;
        }

        .expanded-section:last-child {
            border-bottom: none;
            margin-bottom: 0;
        }

        /* FIXED: Hide mobile tabs on desktop by default */
        .mobile-tabs {
            display: none;
        }

        /* Mobile Responsive */
        @media (max-width: 768px) {
            .main-container {
                display: flex !important;
                flex-direction: column;
                height: 100%;
                grid-template-areas: none;
                grid-template-columns: none;
            }

            .mobile-tabs {
                display: flex;
                background: var(--bg-tertiary);
                border-bottom: 1px solid var(--border-color);
                flex-shrink: 0;
            }

            .header-controls {
                display: none;
            }

            .mobile-tab {
                flex: 1;
                padding: 12px;
                text-align: center;
                background: var(--bg-secondary);
                border-right: 1px solid var(--border-color);
                cursor: pointer;
                transition: all 0.2s;
                font-size: 14px;
            }

            .mobile-tab:last-child {
                border-right: none;
            }

            .mobile-tab.active {
                background: var(--accent-blue);
                color: white;
            }

            .sidebar,
            .progress-panel {
                flex: 1;
                border-right: none;
                border-left: none;
                border-bottom: 1px solid var(--border-color);
                min-height: 0;
                max-height: none;
            }

            .chat-area {
                flex: 1;
                min-height: 0;
            }

            .sidebar,
            .chat-area,
            .progress-panel {
                display: none;
            }
        }

        @media (min-width: 769px) {
            .main-container {
                display: grid !important;
            }

            .sidebar,
            .chat-area,
            .progress-panel {
                display: flex !important;
                height: 100%;
            }
        }

        .agents-list {
            flex: 1;
            overflow-y: auto;
            padding: 16px;
            min-height: 0;
        }

        .agents-header {
            font-size: 14px;
            font-weight: 600;
            color: var(--text-secondary);
            margin-bottom: 12px;
            text-transform: uppercase;
            letter-spacing: 0.5px;
        }

        .agent-item {
            padding: 12px;
            margin-bottom: 8px;
            background: var(--bg-tertiary);
            border: 1px solid var(--border-color);
            border-radius: 8px;
            cursor: pointer;
            transition: all 0.2s;
        }

        .agent-item:hover {
            border-color: var(--accent-blue);
            transform: translateY(-1px);
        }

        .agent-item.active {
            border-color: var(--accent-blue);
            background: rgba(88, 166, 255, 0.1);
        }

        .agent-name {
            font-weight: 600;
            color: var(--text-primary);
            margin-bottom: 4px;
        }

        .agent-description {
            font-size: 12px;
            color: var(--text-muted);
            margin-bottom: 6px;
        }

        .agent-status {
            display: flex;
            align-items: center;
            gap: 6px;
            font-size: 11px;
        }

        .agent-status.online { color: var(--accent-green); }
        .agent-status.offline { color: var(--accent-red); }

        .chat-header {
            padding: 16px 20px;
            border-bottom: 1px solid var(--border-color);
            background: var(--bg-tertiary);
            flex-shrink: 0;
        }

        .chat-title {
            font-size: 16px;
            font-weight: 600;
            color: var(--text-primary);
            margin-bottom: 4px;
        }

        .chat-subtitle {
            font-size: 12px;
            color: var(--text-muted);
        }

        .messages-container {
            flex: 1;
            overflow-y: auto;
            padding: 20px;
            display: flex;
            flex-direction: column;
            gap: 16px;
            min-height: 0;
        }

        .message {
            display: flex;
            gap: 12px;
            max-width: 85%;
        }

        .message.user {
            flex-direction: row-reverse;
            margin-left: auto;
        }

        .message-avatar {
            width: 36px;
            height: 36px;
            border-radius: 50%;
            display: flex;
            align-items: center;
            justify-content: center;
            font-size: 14px;
            font-weight: 600;
            flex-shrink: 0;
        }

        .message.user .message-avatar {
            background: var(--accent-blue);
            color: white;
        }

        .message.agent .message-avatar {
            background: var(--accent-green);
            color: white;
        }

        .message-content {
            padding: 12px 16px;
            border-radius: 16px;
            line-height: 1.5;
            font-size: 14px;
        }

        .message.user .message-content {
            background: var(--accent-blue);
            color: white;
        }

        .message.agent .message-content {
            background: var(--bg-tertiary);
            border: 1px solid var(--border-color);
            color: var(--text-primary);
        }

        /* NEW: Thinking step styles */
        .thinking-step {
            background: var(--bg-secondary);
            border: 1px solid var(--accent-purple);
            border-radius: 12px;
            padding: 12px 16px;
            margin: 8px 0;
            font-size: 13px;
            color: var(--text-secondary);
        }

        .thinking-step.outline-step {
            border-color: var(--accent-cyan);
            background: rgba(57, 208, 216, 0.05);
        }

        .thinking-step-header {
            display: flex;
            align-items: center;
            gap: 8px;
            font-weight: 600;
            margin-bottom: 6px;
            color: var(--text-primary);
        }

        .thinking-step-content {
            line-height: 1.4;
        }

        .message-input {
            border-top: 1px solid var(--border-color);
            padding: 16px 20px;
            display: flex;
            gap: 12px;
            flex-shrink: 0;
            background: var(--bg-secondary);
        }

        .input-field {
            flex: 1;
            background: var(--bg-primary);
            border: 1px solid var(--border-color);
            border-radius: 8px;
            padding: 12px;
            color: var(--text-primary);
            font-size: 14px;
        }

        .input-field:focus {
            outline: none;
            border-color: var(--accent-blue);
        }

        .send-button {
            background: var(--accent-blue);
            color: white;
            border: none;
            border-radius: 8px;
            padding: 12px 20px;
            cursor: pointer;
            font-weight: 600;
            transition: all 0.2s;
        }

        .send-button:hover:not(:disabled) {
            background: #4493f8;
            transform: translateY(-1px);
        }

        .send-button:disabled {
            opacity: 0.5;
            cursor: not-allowed;
            transform: none;
        }

        .panel-header {
            padding: 16px;
            background: var(--bg-tertiary);
            border-bottom: 1px solid var(--border-color);
            font-weight: 600;
            font-size: 14px;
        }

        .panel-content {
            flex: 1;
            overflow-y: auto;
            padding: 16px;
            min-height: 0;
        }

        .progress-section {
            margin-bottom: 20px;
        }

        .section-title {
            font-size: 12px;
            font-weight: 600;
            color: var(--text-muted);
            text-transform: uppercase;
            margin-bottom: 8px;
            letter-spacing: 0.5px;
        }

        /* NEW: Enhanced progress item styles */
        .progress-item {
            background: var(--bg-primary);
            border: 1px solid var(--border-color);
            border-radius: 8px;
            padding: 12px;
            margin-bottom: 8px;
            font-size: 12px;
            transition: all 0.2s;
        }

        .progress-item:hover {
            border-color: var(--accent-blue);
            transform: translateY(-1px);
        }

        .progress-item-header {
            display: flex;
            align-items: center;
            gap: 8px;
            margin-bottom: 6px;
        }

        .progress-icon {
            width: 16px;
            text-align: center;
            font-size: 14px;
        }

        .progress-title {
            font-weight: 500;
            color: var(--text-primary);
            flex: 1;
        }

        .progress-status {
            padding: 2px 6px;
            border-radius: 3px;
            font-size: 10px;
            font-weight: 500;
        }

        .progress-status.running {
            background: var(--accent-orange);
            color: white;
        }

        .progress-status.completed {
            background: var(--accent-green);
            color: white;
        }

        .progress-status.error {
            background: var(--accent-red);
            color: white;
        }

        .progress-status.starting {
            background: var(--accent-cyan);
            color: white;
        }

        .progress-details {
            color: var(--text-secondary);
            font-size: 11px;
            line-height: 1.3;
        }

        .performance-metrics {
            background: rgba(88, 166, 255, 0.05);
            border: 1px solid rgba(88, 166, 255, 0.2);
            border-radius: 6px;
            padding: 8px;
            margin-top: 6px;
            font-size: 10px;
        }

        .performance-metrics .metric {
            display: flex;
            justify-content: space-between;
            margin-bottom: 2px;
        }

        .no-agent-selected {
            display: flex;
            align-items: center;
            justify-content: center;
            flex-direction: column;
            gap: 16px;
            height: 100%;
            color: var(--text-muted);
            text-align: center;
        }

        .no-agent-selected .icon {
            font-size: 48px;
            opacity: 0.5;
        }

        .typing-indicator {
            display: none;
            align-items: center;
            gap: 8px;
            padding: 12px 16px;
            background: var(--bg-tertiary);
            margin: 12px 20px;
            border-radius: 16px;
            font-size: 14px;
            color: var(--text-muted);
            flex-shrink: 0;
        }

        .typing-indicator.active { display: flex; }

        .typing-dots {
            display: flex;
            gap: 4px;
        }

        .typing-dot {
            width: 6px;
            height: 6px;
            border-radius: 50%;
            background: var(--text-muted);
            animation: typing 1.4s infinite;
        }

        .typing-dot:nth-child(2) { animation-delay: 0.2s; }
        .typing-dot:nth-child(3) { animation-delay: 0.4s; }

        @keyframes typing {
            0%, 60%, 100% { opacity: 0.3; }
            30% { opacity: 1; }
        }

        .system-info {
            margin-top: auto;
            padding: 12px;
            border-top: 1px solid var(--border-color);
            font-size: 11px;
            color: var(--text-muted);
            flex-shrink: 0;
        }

        .error-message {
            background: rgba(248, 81, 73, 0.1);
            border: 1px solid var(--accent-red);
            color: var(--accent-red);
            padding: 12px;
            border-radius: 6px;
            margin: 12px;
            font-size: 14px;
            position: fixed;
            bottom: 20px;
            right: 20px;
            z-index: 2000;
            max-width: 300px;
        }

        .event-detail-modal {
            position: fixed;
            top: 0;
            left: 0;
            right: 0;
            bottom: 0;
            background: rgba(0, 0, 0, 0.8);
            display: none;
            align-items: center;
            justify-content: center;
            z-index: 2000;
            padding: 20px;
        }

        .event-detail-modal.active {
            display: flex;
        }

        .event-detail-content {
            background: var(--bg-secondary);
            border: 1px solid var(--border-color);
            border-radius: 12px;
            max-width: 800px;
            max-height: 80vh;
            width: 100%;
            display: flex;
            flex-direction: column;
            overflow: hidden;
            box-shadow: var(--shadow);
        }

        .event-detail-header {
            padding: 20px 24px;
            border-bottom: 1px solid var(--border-color);
            background: var(--bg-tertiary);
            display: flex;
            align-items: center;
            justify-content: space-between;
            flex-shrink: 0;
        }

        .event-detail-title {
            font-size: 18px;
            font-weight: 600;
            color: var(--text-primary);
            display: flex;
            align-items: center;
            gap: 12px;
        }

        .event-detail-close {
            background: none;
            border: none;
            color: var(--text-muted);
            cursor: pointer;
            padding: 8px;
            border-radius: 6px;
            font-size: 20px;
            transition: all 0.2s;
        }

        .event-detail-close:hover {
            background: var(--bg-primary);
            color: var(--text-primary);
        }

        .event-detail-body {
            flex: 1;
            overflow-y: auto;
            padding: 24px;
            min-height: 0;
        }

        .event-section {
            margin-bottom: 24px;
        }

        .event-section-title {
            font-size: 14px;
            font-weight: 600;
            color: var(--accent-blue);
            text-transform: uppercase;
            letter-spacing: 0.5px;
            margin-bottom: 12px;
            padding-bottom: 6px;
            border-bottom: 1px solid var(--border-color);
        }

        .event-field {
            display: flex;
            justify-content: space-between;
            align-items: flex-start;
            padding: 8px 0;
            border-bottom: 1px solid rgba(48, 54, 61, 0.5);
            font-size: 14px;
        }

        .event-field:last-child {
            border-bottom: none;
        }

        .event-field-label {
            font-weight: 500;
            color: var(--text-secondary);
            min-width: 140px;
            flex-shrink: 0;
        }

        .event-field-value {
            color: var(--text-primary);
            flex: 1;
            text-align: right;
            word-break: break-word;
        }

        .event-field-value.json {
            background: var(--bg-primary);
            border-radius: 6px;
            padding: 8px;
            font-family: 'Consolas', 'Monaco', monospace;
            font-size: 12px;
            text-align: left;
            white-space: pre-wrap;
            max-height: 200px;
            overflow-y: auto;
        }

        .event-status-badge {
            padding: 4px 8px;
            border-radius: 4px;
            font-size: 12px;
            font-weight: 500;
        }

        .event-status-badge.completed {
            background: var(--accent-green);
            color: white;
        }

        .event-status-badge.running {
            background: var(--accent-orange);
            color: white;
        }

        .event-status-badge.failed {
            background: var(--accent-red);
            color: white;
        }

        .progress-item {
            cursor: pointer;
            transition: all 0.2s;
        }

        .progress-item:hover {
            transform: translateY(-2px);
            box-shadow: 0 4px 12px rgba(0, 0, 0, 0.2);
        }

        .thinking-step {
            cursor: pointer;
            transition: all 0.2s;
        }

        .thinking-step:hover {
            transform: translateY(-1px);
            box-shadow: 0 2px 8px rgba(0, 0, 0, 0.15);
        }
    </style>
</head>
<body>

<div class="api-key-modal" id="api-key-modal">
    <div class="api-key-content">
        <div class="api-key-title">🔐 Enter API Key</div>
        <div class="api-key-description">
            Please enter your API key to access the agent. You can find this key in your agent registration details.
        </div>
        <input type="text" class="api-key-input" id="api-key-input"
               placeholder="tbk_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx">
        <button class="api-key-button" id="api-key-submit">Connect</button>
    </div>
</div>

<div class="header">
    <div class="logo">
        <span>🤖</span>
        <span>Agent Registry</span>
    </div>
    <div class="header-controls">
        <button class="panel-toggle active" id="sidebar-toggle">📋 Agents</button>
        <button class="panel-toggle active" id="progress-toggle">📊 Progress</button>
        <div class="status-indicator disconnected" id="connection-status">
            <div class="status-dot"></div>
            <span>Connecting...</span>
        </div>
    </div>
</div>

<div class="mobile-tabs">
    <div class="mobile-tab active" data-tab="chat">💬 Chat</div>
    <div class="mobile-tab" data-tab="agents">📋 Agents</div>
    <div class="mobile-tab" data-tab="progress">📊 Progress</div>
</div>

<div class="main-container">
    <!-- Agents Sidebar -->
    <div class="sidebar" id="sidebar">
        <div class="sidebar-header">
            <div class="sidebar-title">Available Agents</div>
            <button class="collapse-btn" id="sidebar-collapse">◀</button>
        </div>
        <div class="agents-list">
            <div id="agents-container">
                <div style="color: var(--text-muted); font-size: 12px; text-align: center; padding: 20px;">
                    Loading agents...
                </div>
            </div>
        </div>
        <div class="system-info">
            <div>Registry Server</div>
            <div id="server-info">ws://localhost:8080</div>
        </div>
    </div>

    <!-- Chat Area -->
    <div class="chat-area">
        <div class="chat-header">
            <div class="chat-title" id="chat-title">Select an Agent</div>
            <div class="chat-subtitle" id="chat-subtitle">Choose an agent from the sidebar to start chatting</div>
        </div>

        <div class="messages-container" id="messages-container">
            <div class="no-agent-selected">
                <div class="icon">💬</div>
                <div>Select an agent to start a conversation</div>
            </div>
        </div>

        <div class="typing-indicator" id="typing-indicator">
            <span>Agent is thinking</span>
            <div class="typing-dots">
                <div class="typing-dot"></div>
                <div class="typing-dot"></div>
                <div class="typing-dot"></div>
            </div>
        </div>

        <div class="message-input">
            <input type="text" class="input-field" id="message-input"
                   placeholder="Type your message..." disabled>
            <button class="send-button" id="send-button" disabled>Send</button>
        </div>
    </div>
    <!-- Progress Panel -->
    <div class="progress-panel" id="progress-panel">
        <div class="progress-header">
            <span>Live Progress</span>
            <button class="collapse-btn" id="progress-collapse">▶</button>
        </div>
        <div class="panel-content" id="progress-content">
            <div class="progress-section">
                <div class="section-title">Current Status</div>
                <div id="current-status">
                    <div style="color: var(--text-muted); font-size: 12px; text-align: center; padding: 20px;">
                        No active execution
                    </div>
                </div>
            </div>

            <div class="progress-section">
                <div class="section-title">Performance Metrics</div>
                <div id="performance-metrics">
                    <div style="color: var(--text-muted); font-size: 12px; text-align: center; padding: 10px;">
                        No metrics available
                    </div>
                </div>
            </div>

            <div class="progress-section">
                <div class="section-title">Meta Tools History</div>
                <div id="meta-tools-history">
                    <div style="color: var(--text-muted); font-size: 12px; text-align: center; padding: 10px;">
                        No meta-tool activity
                    </div>
                </div>
            </div>

            <div class="progress-section">
                <div class="section-title">System Events</div>
                <div id="system-events">
                    <div style="color: var(--text-muted); font-size: 12px; text-align: center; padding: 10px;">
                        System idle
                    </div>
                </div>
            </div>
        </div>
    </div>
</div>

<script unSave="true">



    class AgentRegistryUI {
        constructor() {
            this.ws = null;
            this.currentAgent = null;
            this.sessionId = 'ui_session_' + Math.random().toString(36).substr(2, 9);
            this.isConnected = false;
            this.reconnectAttempts = 0;
            this.apiKey = null;
            this.maxReconnectAttempts = 10;
            this.reconnectDelay = 1000;

            this.panelStates = {
                sidebar: true,
                progress: true,
                mobile: 'chat'
            };

            this.agents = new Map();
            this.currentExecution = null;

            // NEW: Enhanced progress tracking
            this.progressHistory = [];
            this.maxProgressHistory = 200;
            this.expandedProgressItem = null;
            this.currentPerformanceMetrics = null;
            this.currentOutline = null;

            this.elements = {
                connectionStatus: document.getElementById('connection-status'),
                agentsContainer: document.getElementById('agents-container'),
                chatTitle: document.getElementById('chat-title'),
                chatSubtitle: document.getElementById('chat-subtitle'),
                messagesContainer: document.getElementById('messages-container'),
                messageInput: document.getElementById('message-input'),
                sendButton: document.getElementById('send-button'),
                typingIndicator: document.getElementById('typing-indicator'),
                serverInfo: document.getElementById('server-info'),

                // API Key elements
                apiKeyModal: document.getElementById('api-key-modal'),
                apiKeyInput: document.getElementById('api-key-input'),
                apiKeySubmit: document.getElementById('api-key-submit'),

                // Panel control elements
                sidebarToggle: document.getElementById('sidebar-toggle'),
                progressToggle: document.getElementById('progress-toggle'),
                sidebarCollapse: document.getElementById('sidebar-collapse'),
                progressCollapse: document.getElementById('progress-collapse'),
                mainContainer: document.querySelector('.main-container'),
                sidebar: document.getElementById('sidebar'),
                progressPanel: document.getElementById('progress-panel'),
                progressContent: document.getElementById('progress-content')
            };

            // Enhanced cleanup timer
            setInterval(() => {
                if (this.isTyping && this.currentExecution) {
                    const timeSinceLastUpdate = Date.now() - this.currentExecution.lastUpdate;
                    if (timeSinceLastUpdate > 30000) {
                        console.log('🧹 Cleanup: Hiding stuck typing indicator');
                        this.showTypingIndicator(false);
                        this.currentExecution = null;
                        this.updateCurrentStatusToIdle();
                    }
                }
            }, 5000);

            this.init();
        }

        init() {
            this.setupEventListeners();
            this.setupPanelControls();
            this.initializeProgressPanel();
            this.showApiKeyModal();
        }

        // NEW: Initialize the refactored progress panel
        initializeProgressPanel() {
            if (this.elements.progressContent) {
                this.elements.progressContent.innerHTML = `
                <div class="progress-section metrics-section">
                    <div class="section-title expandable-section" onclick="window.agentUI.toggleSection('metrics')">
                        <span>📊 Performance Metrics</span>
                        <span class="section-toggle">▼</span>
                    </div>
                    <div class="section-content" id="performance-metrics">
                        <div class="no-data">No metrics available</div>
                    </div>
                </div>

                <div class="progress-section outline-section">
                    <div class="section-title expandable-section" onclick="window.agentUI.toggleSection('outline')">
                        <span>🗺️ Execution Outline & Context</span>
                        <span class="section-toggle">▼</span>
                    </div>
                    <div class="section-content" id="execution-outline">
                        <div class="no-data">No outline available</div>
                    </div>
                </div>

                <div class="progress-section status-history-section">
                    <div class="section-title expandable-section" onclick="window.agentUI.toggleSection('status')">
                        <span>⚡ Status & History</span>
                        <span class="section-toggle">▼</span>
                    </div>
                    <div class="section-content expanded" id="status-history">
                        <div class="no-data">No active execution</div>
                    </div>
                </div>
            `;
            }
        }

        // NEW: Toggle progress panel sections
        toggleSection(sectionName) {
            const section = document.querySelector(`.${sectionName}-section .section-content`);
            const toggle = document.querySelector(`.${sectionName}-section .section-toggle`);

            if (!section || !toggle) return;

            const isExpanded = section.classList.contains('expanded');

            if (isExpanded) {
                section.classList.remove('expanded');
                toggle.textContent = '▼';
            } else {
                section.classList.add('expanded');
                toggle.textContent = '▲';
            }
        }

        // REFACTORED: Main message handler with unified progress system
        handleWebSocketMessage(data) {
            console.log('WebSocket message received:', data);

            if (data.event === 'execution_progress') {
                const executionData = data.data;
                if (executionData && executionData.payload) {
                    this.handleUnifiedProgressEvent(executionData);
                }
                return;
            }

            if (data.request_id && data.payload) {
                this.handleUnifiedProgressEvent(data);
                return;
            }

            if (data.event) {
                this.handleRegistryEvent(data);
                return;
            }

            console.log('Unhandled message format:', data);
        }

        // NEW: Unified progress event handler
        // REPLACE the existing handleUnifiedProgressEvent method
        handleUnifiedProgressEvent(eventData) {
            const payload = eventData.payload;
            const eventType = payload.event_type;
            const isFinal = eventData.is_final;
            const requestId = eventData.request_id;

            console.log(`🎯 Processing Event: ${eventType}`, payload);

            // Handle final events
            if (isFinal || eventType === 'execution_complete' || payload.status === 'completed') {
                this.showTypingIndicator(false);

                const result = payload.metadata?.result || payload.result || payload.response || payload.output;
                if (result && typeof result === 'string' && result.trim()) {
                    this.addMessage('agent', result);
                }

                this.currentExecution = null;
                this.updateCurrentStatusToIdle();
                return;
            }

            // Initialize execution tracking
            if (!this.currentExecution) {
                this.currentExecution = {
                    requestId,
                    startTime: Date.now(),
                    events: [],
                    lastUpdate: Date.now()
                };
                this.showTypingIndicator(true);
            }

            // ADD: Store ALL events in progress history
            this.addToProgressHistory(payload);

            // Handle chat integration for important events
            this.handleChatIntegration(payload);

            // Update performance metrics
            this.updatePerformanceMetricsFromEvent(payload);

            // Update execution outline
            this.updateExecutionOutlineFromEvent(payload);

            // Refresh status history (shows all events)
            this.refreshStatusHistory();

            // Update current execution
            if (this.currentExecution) {
                this.currentExecution.events.push({...payload, timestamp: Date.now()});
                this.currentExecution.lastUpdate = Date.now();
            }
        }

        // NEW: Add event to progress history
        // UPDATE the addToProgressHistory method to ensure all events are captured
        addToProgressHistory(payload) {
        const irrelevantEventTypes = ['node_phase', 'node_enter']; // Fügen Sie hier weitere Typen hinzu, falls nötig

    // Prüfen, ob der Event-Typ in der Liste der irrelevanten Typen ist
    if (irrelevantEventTypes.includes(payload.event_type)) {
        // Optional: Hier könnte man das Event kurz an anderer Stelle anzeigen,
        // aber wir speichern es nicht im langfristigen Verlauf.
        console.log(`📝 Skipping storage for irrelevant event: ${payload.event_type}`);
        return; // Die Funktion hier beenden, um das Speichern zu verhindern
    }

            // Generate consistent ID for events
            const eventId = payload.event_id || `event_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;

            const historyItem = {
                ...payload,
                timestamp: payload.timestamp || Date.now(),
                id: eventId
            };

            // Remove any existing event with same ID to avoid duplicates
            this.progressHistory = this.progressHistory.filter(item => item.id !== eventId);

            this.progressHistory.unshift(historyItem);

            if (this.progressHistory.length - 10 > this.maxProgressHistory) {
                this.progressHistory = this.progressHistory.slice(0, this.maxProgressHistory-50);
            }

            console.log(`📝 Added to progress history: ${payload.event_type}`, historyItem);
        }

        // NEW: Refresh unified status history display
        refreshStatusHistory() {
            const container = document.getElementById('status-history');
            if (!container) return;

            if (this.progressHistory.length === 0) {
                container.innerHTML = '<div class="no-data">No events recorded</div>';
                return;
            }

            container.innerHTML = '';

            this.progressHistory.forEach((event, index) => {
                const eventElement = this.createExpandableProgressItem(event, index === 0);
                container.appendChild(eventElement);
            });
        }

        // NEW: Create expandable progress item (only one expandable at a time)
// UPDATE the createExpandableProgressItem method to show more LLM details
        createExpandableProgressItem(event, isLatest = false) {
            const div = document.createElement('div');
            div.className = `progress-item expandable ${isLatest ? 'latest' : ''} ${event.event_type}`;
            div.setAttribute('data-event-id', event.id);

            const icon = this.getEventIcon(event.event_type, event.status);
            const title = this.getDisplayAction(event.event_type, event);
            const timestamp = new Date((event.timestamp || Date.now()) * (event.timestamp > 10000000000 ? 1 : 1000)).toLocaleTimeString();
            const status = event.status || 'unknown';

            // ADD: Special summary for LLM calls
            let summaryDetails = '';
            if (event.node_name) summaryDetails += `${event.node_name} • `;
            summaryDetails += timestamp;

            if (event.event_type === 'llm_call') {
                if (event.llm_temperature !== undefined) summaryDetails += ` • Temp: ${event.llm_temperature}`;
                if (event.llm_total_tokens) summaryDetails += ` • ${event.llm_total_tokens} tokens`;
                if (event.llm_cost) summaryDetails += ` • $${event.llm_cost.toFixed(4)}`;
                if (event.duration) summaryDetails += ` • ${event.duration.toFixed(2)}s`;
            } else {
                if (event.duration) summaryDetails += ` • ${event.duration.toFixed(2)}s`;
            }

            div.innerHTML = `
        <div class="progress-item-header" onclick="window.agentUI.toggleProgressItem('${event.id}')">
            <div class="progress-icon">${icon}</div>
            <div class="progress-title">${title}</div>
            <div class="progress-meta">
                <span class="progress-status ${status}">${status}</span>
                <span class="expand-indicator">▼</span>
            </div>
        </div>
        <div class="progress-summary">
            ${summaryDetails}
        </div>
        <div class="progress-item-expanded" id="expanded-${event.id}">
            ${this.createExpandedEventContent(event)}
        </div>
    `;

            return div;
        }
        // NEW: Toggle progress item (only one at a time)
        toggleProgressItem(eventId) {
            if (this.expandedProgressItem && this.expandedProgressItem !== eventId) {
                this.closeProgressItem(this.expandedProgressItem);
            }

            const expandedContent = document.getElementById(`expanded-${eventId}`);
            const progressItem = document.querySelector(`[data-event-id="${eventId}"]`);
            const indicator = progressItem?.querySelector('.expand-indicator');

            if (!expandedContent || !progressItem) return;

            const isExpanded = expandedContent.classList.contains('active');

            if (isExpanded) {
                expandedContent.classList.remove('active');
                progressItem.classList.remove('expanded');
                if (indicator) indicator.textContent = '▼';
                this.expandedProgressItem = null;
            } else {
                expandedContent.classList.add('active');
                progressItem.classList.add('expanded');
                if (indicator) indicator.textContent = '▲';
                this.expandedProgressItem = eventId;
            }
        }

        // NEW: Close progress item
        closeProgressItem(eventId) {
            const expandedContent = document.getElementById(`expanded-${eventId}`);
            const progressItem = document.querySelector(`[data-event-id="${eventId}"]`);
            const indicator = progressItem?.querySelector('.expand-indicator');

            if (expandedContent) expandedContent.classList.remove('active');
            if (progressItem) progressItem.classList.remove('expanded');
            if (indicator) indicator.textContent = '▼';
        }

        // NEW: Create detailed expanded content
// ADD this method to create comprehensive event details
        createExpandedEventContent(event) {
            const sections = [];

            // Core Information
            const coreInfo = this.extractCoreFields(event);
            if (Object.keys(coreInfo).length > 0) {
                sections.push(this.createEventSection('Core Information', coreInfo));
            }

            // Timing Information
            const timingInfo = this.extractTimingFields(event);
            if (Object.keys(timingInfo).length > 0) {
                sections.push(this.createEventSection('Timing & Status', timingInfo));
            }

            // LLM Information
            const llmInfo = this.extractLLMFields(event);
            if (Object.keys(llmInfo).length > 0) {
                sections.push(this.createEventSection('LLM Details', llmInfo));
            }

            // Tool Information
            const toolInfo = this.extractToolFields(event);
            if (Object.keys(toolInfo).length > 0) {
                sections.push(this.createEventSection('Tool Details', toolInfo));
            }

            // Performance Information
            const perfInfo = this.extractPerformanceFields(event);
            if (Object.keys(perfInfo).length > 0) {
                sections.push(this.createEventSection('Performance', perfInfo));
            }

            // Reasoning Context
            const reasoningInfo = this.extractReasoningFields(event);
            if (Object.keys(reasoningInfo).length > 0) {
                sections.push(this.createEventSection('Reasoning Context', reasoningInfo));
            }

            // Error Information
            const errorInfo = this.extractErrorFields(event);
            if (Object.keys(errorInfo).length > 0) {
                sections.push(this.createEventSection('Error Details', errorInfo));
            }

            // Raw Data
            const rawData = this.extractRawDataFields(event);
            if (Object.keys(rawData).length > 0) {
                sections.push(this.createEventSection('Raw Data', rawData));
            }

            return sections.join('') || '<div class="no-expanded-data">No detailed information available</div>';
        }

        // NEW: Create event section for expanded view
        createEventSection(title, fields) {
            const fieldsHtml = Object.entries(fields)
                .map(([key, value]) => {
                    if (typeof value === 'object' && value.type === 'json') {
                        return `
                        <div class="event-field">
                            <div class="event-field-label">${key}:</div>
                            <div class="event-field-value json">${value.value}</div>
                        </div>
                    `;
                    } else {
                        return `
                        <div class="event-field">
                            <div class="event-field-label">${key}:</div>
                            <div class="event-field-value">${value}</div>
                        </div>
                    `;
                    }
                })
                .join('');

            return `
            <div class="expanded-section">
                <div class="expanded-section-title">${title}</div>
                ${fieldsHtml}
            </div>
        `;
        }

        // ENHANCED: Chat integration for reasoning loops and task execution
        handleChatIntegration(payload) {
            const eventType = payload.event_type;
            const metadata = payload.metadata || {};
            switch (eventType) {
                case 'reasoning_loop':
                    if (metadata.outline_step && metadata.outline_total) {
                        this.handleOutlineStepInChat(payload);
                    }
                    break;
                case 'outline_created':
                    this.handleOutlineCreatedInChat(payload);
                    break;
                case 'task_start':
                case 'task_complete':
                case 'task_error':
                    this.handleTaskProgressInChat(payload);
                    break;
                case 'plan_created':
                    this.handlePlanCreatedInChat(payload);
                    break;
                case 'tool_call':
                    // Only show important tool calls in chat
                    if (payload.tool_name && !payload.tool_name.includes('internal')) {
                        this.handleToolCallInChat(payload);
                    }
                    break;
            }
        }

        // ADD this method for plan creation
handlePlanCreatedInChat(payload) {
    const metadata = payload.metadata || {};
    const planName = metadata.plan_name || 'Execution Plan';
    const taskCount = metadata.task_count || 0;
    const strategy = metadata.strategy || 'sequential';

    const planDiv = document.createElement('div');
    planDiv.className = 'thinking-step plan-created';
    planDiv.innerHTML = `
        <div class="thinking-step-header">
            <span>📋</span>
            <span>${planName} Created</span>
            <span class="step-status completed">Ready</span>
        </div>
        <div class="thinking-step-content">
            <div class="plan-details">
                <div class="plan-info">
                    <span class="plan-item">Tasks: ${taskCount}</span>
                    <span class="plan-item">Strategy: ${strategy}</span>
                </div>
                <div class="plan-ready">
                    <em>🚀 Plan ready for execution</em>
                </div>
            </div>
        </div>
    `;

    this.elements.messagesContainer.appendChild(planDiv);
    this.scrollToBottom();
}

        // ADD this new method for outline step progress
handleOutlineStepInChat(payload) {
    const metadata = payload.metadata || {};
    const outlineStep = metadata.outline_step || 0;
    const outlineTotal = metadata.outline_total || 0;
    const loopNumber = metadata.loop_number || 0;
    const status = payload.status || 'running';

    if (outlineStep === 0 || outlineTotal === 0) return;

    const progressPercentage = Math.round((outlineStep / outlineTotal) * 100);
    const isCompleted = status === 'completed';

    const stepDiv = document.createElement('div');
    stepDiv.className = `thinking-step outline-step ${isCompleted ? 'completed' : 'running'}`;

    let stepTitle = `Outline Step ${outlineStep} of ${outlineTotal}`;
    let stepIcon = isCompleted ? '✅' : '🗺️';
    let stepStatus = isCompleted ? 'Completed' : 'In Progress';

    stepDiv.innerHTML = `
        <div class="thinking-step-header">
            <span>${stepIcon}</span>
            <span>${stepTitle}</span>
            <span class="step-status ${status}">${stepStatus}</span>
        </div>
        <div class="thinking-step-content">
            <div class="outline-progress">
                <div class="progress-info">
                    <span class="progress-text">Execution Progress</span>
                    <span class="progress-percentage">${progressPercentage}%</span>
                </div>
                <div class="progress-bar-container">
                    <div class="progress-bar">
                        <div class="progress-bar-fill" style="width: ${progressPercentage}%"></div>
                    </div>
                </div>
                ${isCompleted ?
                    '<div class="step-completed">This execution step is now complete</div>' :
                    '<div class="step-working">Working on this step...</div>'
                }
            </div>

            ${metadata.context_size || metadata.task_stack_size ? `
                <div class="context-info">
                    ${metadata.context_size ? `<span class="context-item">Context: ${metadata.context_size}</span>` : ''}
                    ${metadata.task_stack_size ? `<span class="context-item">Tasks: ${metadata.task_stack_size}</span>` : ''}
                </div>
            ` : ''}
        </div>
    `;

    this.elements.messagesContainer.appendChild(stepDiv);
    this.scrollToBottom();
}


        // NEW: Handle outline creation with detailed information
// REPLACE the existing handleOutlineCreatedInChat method
handleOutlineCreatedInChat(payload) {
    const metadata = payload.metadata || {};
    const outline = metadata.outline;

    if (!outline) return;

    const outlineDiv = document.createElement('div');
    outlineDiv.className = 'thinking-step outline-created';
    outlineDiv.innerHTML = `
        <div class="thinking-step-header">
            <span>📋</span>
            <span>Execution Plan Created</span>
            <span class="step-status completed">Ready</span>
        </div>
        <div class="thinking-step-content">
            <div class="outline-content">
                ${this.formatOutlineForChat(outline)}
            </div>
            <div class="outline-ready">
                <em>✨ Ready to execute plan step by step</em>
            </div>
        </div>
    `;

    this.elements.messagesContainer.appendChild(outlineDiv);
    this.scrollToBottom();
}

        // NEW: Handle task execution progress cleanly
        handleTaskProgressInChat(payload) {
            const eventType = payload.event_type;
            const taskId = payload.task_id;
            const metadata = payload.metadata || {};
            const description = metadata.description || 'Task execution';
            const taskType = metadata.type || 'Task';
            const priority = metadata.priority || 'normal';

            let icon = '📋';
            let status = '';
            let statusClass = 'running';

            if (eventType === 'task_start') {
                icon = '▶️';
                status = 'Starting';
                statusClass = 'starting';
            } else if (eventType === 'task_complete') {
                icon = '✅';
                status = 'Completed';
                statusClass = 'completed';
            } else if (eventType === 'task_error') {
                icon = '❌';
                status = 'Failed';
                statusClass = 'error';
            }

            const taskDiv = document.createElement('div');
            taskDiv.className = `thinking-step task-progress ${statusClass}`;
            taskDiv.innerHTML = `
            <div class="thinking-step-header">
                <span>${icon}</span>
                <span>${taskType} ${status}</span>
                <span class="priority-badge ${priority}">${priority}</span>
            </div>
            <div class="thinking-step-content">
                <div class="task-description">${description}</div>
                ${payload.duration ? `<div class="task-timing">Duration: ${payload.duration.toFixed(2)}s</div>` : ''}
                ${eventType === 'task_error' && payload.error_details?.message ?
                `<div class="task-error">Error: ${payload.error_details.message}</div>` : ''}
            </div>
        `;

            this.elements.messagesContainer.appendChild(taskDiv);
            this.scrollToBottom();
        }

        // NEW: Handle tool calls in chat
        handleToolCallInChat(payload) {
            const toolName = payload.tool_name;
            const status = payload.status;

            if (status === 'running') return; // Only show completed tool calls

            const toolDiv = document.createElement('div');
            toolDiv.className = `thinking-step tool-call ${status}`;
            toolDiv.innerHTML = `
            <div class="thinking-step-header">
                <span>🔧</span>
                <span>Used ${toolName}</span>
                <span class="tool-status ${status}">${status}</span>
            </div>
            <div class="thinking-step-content">
                <div class="tool-result">
                    ${status === 'completed' ? 'Tool executed successfully' : 'Tool execution failed'}
                    ${payload.duration ? ` in ${payload.duration.toFixed(2)}s` : ''}
                </div>
            </div>
        `;

            this.elements.messagesContainer.appendChild(toolDiv);
            this.scrollToBottom();
        }

        // NEW: Format outline for chat display
        formatOutlineForChat(outline) {
            if (typeof outline === 'string') {
                return `<div class="outline-text">${outline}</div>`;
            }

            if (Array.isArray(outline)) {
                return `
                <div class="outline-steps">
                    ${outline.map((step, index) =>
                    `<div class="outline-step">
                            <span class="step-number">${index + 1}.</span>
                            <span class="step-text">${step}</span>
                        </div>`
                ).join('')}
                </div>
            `;
            }

            return '<div class="outline-text">Execution plan created</div>';
        }

        // NEW: Create progress bar
        createProgressBar(current, total) {
            if (!total || total === 0) return '';

            const percentage = Math.round((current / total) * 100);

            return `
            <div class="progress-bar-container">
                <div class="progress-bar-info">
                    <span>Progress</span>
                    <span>${percentage}%</span>
                </div>
                <div class="progress-bar">
                    <div class="progress-bar-fill" style="width: ${percentage}%"></div>
                </div>
            </div>
        `;
        }

        // ENHANCED: Update performance metrics
        updatePerformanceMetricsFromEvent(payload) {
            const metadata = payload.metadata || {};
            const performance = metadata.performance_metrics;

            if (performance && Object.keys(performance).length > 0) {
                this.currentPerformanceMetrics = performance;
                this.refreshPerformanceMetrics();
            }
        }

        // NEW: Refresh performance metrics display
        refreshPerformanceMetrics() {
            const container = document.getElementById('performance-metrics');
            if (!container || !this.currentPerformanceMetrics) return;

            const metrics = {
                'Action Efficiency': `${Math.round((this.currentPerformanceMetrics.action_efficiency || 0) * 100)}%`,
                'Avg Loop Time': `${(this.currentPerformanceMetrics.avg_loop_time || 0).toFixed(1)}s`,
                'Progress Rate': `${Math.round((this.currentPerformanceMetrics.progress_rate || 0) * 100)}%`,
                'Total Loops': this.currentPerformanceMetrics.total_loops || 0,
                'Progress Loops': this.currentPerformanceMetrics.progress_loops || 0
            };

            container.innerHTML = `
            <div class="metrics-grid">
                ${Object.entries(metrics).map(([key, value]) => `
                    <div class="metric-card">
                        <div class="metric-label">${key}</div>
                        <div class="metric-value">${value}</div>
                    </div>
                `).join('')}
            </div>
        `;
        }

        // NEW: Update execution outline
        updateExecutionOutlineFromEvent(payload) {
            const eventType = payload.event_type;
            const metadata = payload.metadata || {};

            if (eventType === 'outline_created' || eventType === 'reasoning_loop') {
                const outlineContainer = document.getElementById('execution-outline');
                if (!outlineContainer) return;

                const outline = metadata.outline;
                const outlineStep = metadata.outline_step || 0;
                const outlineTotal = metadata.outline_total || 0;
                const contextSize = metadata.context_size || 0;
                const taskStackSize = metadata.task_stack_size || 0;

                outlineContainer.innerHTML = `
                <div class="outline-info">
                    <div class="context-metrics">
                        <div class="context-metric">
                            <span class="context-label">Context Size:</span>
                            <span class="context-value">${contextSize}</span>
                        </div>
                        <div class="context-metric">
                            <span class="context-label">Task Stack:</span>
                            <span class="context-value">${taskStackSize}</span>
                        </div>
                        <div class="context-metric">
                            <span class="context-label">Progress:</span>
                            <span class="context-value">${outlineStep}/${outlineTotal}</span>
                        </div>
                    </div>

                    ${outlineTotal > 0 ? this.createProgressBar(outlineStep, outlineTotal) : ''}
                </div>

                ${outline ? `
                    <div class="outline-details">
                        <div class="outline-title">Current Plan</div>
                        ${this.formatOutlineForChat(outline)}
                    </div>
                ` : ''}
            `;
            }
        }

        // Helper methods for field extraction (using existing implementations)
        extractCoreFields(event) {
            const fields = {};
            if (event.event_type) fields['Event Type'] = event.event_type.replace(/_/g, ' ').toUpperCase();
            if (event.node_name) fields['Node'] = event.node_name;
            if (event.agent_name) fields['Agent'] = event.agent_name;
            if (event.task_id) fields['Task ID'] = event.task_id;
            if (event.plan_id) fields['Plan ID'] = event.plan_id;
            if (event.timestamp) fields['Timestamp'] = new Date((event.timestamp > 10000000000 ? event.timestamp : event.timestamp * 1000)).toLocaleString();
            return fields;
        }

// REPLACE the existing extractLLMFields method
        extractLLMFields(event) {
            const fields = {};
            const metadata = event.metadata || {};

            if (event.llm_model) fields['Model'] = event.llm_model;
            if (event.llm_temperature !== undefined) fields['Temperature'] = event.llm_temperature;
            if (event.llm_prompt_tokens) fields['Prompt Tokens'] = event.llm_prompt_tokens.toLocaleString();
            if (event.llm_completion_tokens) fields['Completion Tokens'] = event.llm_completion_tokens.toLocaleString();
            if (event.llm_total_tokens) fields['Total Tokens'] = event.llm_total_tokens.toLocaleString();
            if (event.llm_cost) fields['Cost'] = `$${event.llm_cost.toFixed(4)}`;

            // ADD: Model preferences and metadata
            if (metadata.model_preference) fields['Model Preference'] = metadata.model_preference;

            return fields;
        }

        extractToolFields(event) {
            const fields = {};
            const metadata = event.metadata || {};

            if (event.tool_name) fields['Tool Name'] = event.tool_name;
            if (metadata.meta_tool_name) fields['Meta Tool Name'] = metadata.meta_tool_name;

            if (event.is_meta_tool !== null && event.is_meta_tool !== undefined) {
                fields['Is Meta Tool'] = event.is_meta_tool ? '✅ Yes' : '❌ No';
            }

            // ADD: Tool execution details
            if (metadata.execution_phase) fields['Execution Phase'] = metadata.execution_phase;
            if (metadata.reasoning_loop) fields['Reasoning Loop'] = metadata.reasoning_loop;
            if (metadata.parsed_args && metadata.parsed_args.confidence_level) {
                fields['Confidence Level'] = `${Math.round(metadata.parsed_args.confidence_level * 100)}%`;
            }

            return fields;
        }

// ADD these helper methods for comprehensive data extraction
        extractTimingFields(event) {
            const fields = {};

            if (event.status) {
                fields['Status'] = `<span class="event-status-badge ${event.status}">${event.status.toUpperCase()}</span>`;
            }
            if (event.success !== null && event.success !== undefined) {
                fields['Success'] = event.success ? '✅ Yes' : '❌ No';
            }
            if (event.timestamp) {
                fields['Timestamp'] = new Date((event.timestamp > 10000000000 ? event.timestamp : event.timestamp * 1000)).toLocaleString();
            }
            if (event.duration) {
                fields['Duration'] = `${event.duration.toFixed(3)}s`;
            }
            if (event.node_duration) {
                fields['Node Duration'] = `${event.node_duration.toFixed(3)}s`;
            }
            if (event.routing_decision) {
                fields['Next Step'] = event.routing_decision;
            }

            return fields;
        }

        extractErrorFields(event) {
            const fields = {};

            if (event.error_details) {
                const errorDetails = event.error_details;
                if (errorDetails.message) fields['Error Message'] = errorDetails.message;
                if (errorDetails.type) fields['Error Type'] = errorDetails.type;
                if (errorDetails.traceback) {
                    fields['Traceback'] = {
                        type: 'json',
                        value: errorDetails.traceback
                    };
                }
            }

            if (event.tool_error) {
                fields['Tool Error'] = event.tool_error;
            }

            return fields;
        }
// REPLACE the existing extractRawDataFields method
        extractRawDataFields(event) {
            const fields = {};

            // ADD: Full LLM Input/Output for LLM calls
            if (event.event_type === 'llm_call') {
                if (event.llm_input) {
                    fields['LLM Input (Full Prompt)'] = {
                        type: 'json',
                        value: event.llm_input
                    };
                }

                if (event.llm_output) {
                    fields['LLM Output (Response)'] = {
                        type: 'json',
                        value: event.llm_output
                    };
                }
            }

            // Show other raw data for tool calls
            if (event.tool_args && typeof event.tool_args === 'object') {
                fields['Tool Arguments'] = {
                    type: 'json',
                    value: JSON.stringify(event.tool_args, null, 2)
                };
            }

            if (event.tool_result) {
                const resultStr = typeof event.tool_result === 'string' ?
                    event.tool_result :
                    JSON.stringify(event.tool_result, null, 2);

                fields['Tool Result'] = {
                    type: 'json',
                    value: resultStr.length > 1000 ?
                        resultStr.substring(0, 1000) + '\\n\\n... [truncated]' :
                        resultStr
                };
            }

            return fields;
        }

        extractPerformanceFields(event) {
            const fields = {};
            const metadata = event.metadata || {};
            const performance = metadata.performance_metrics || {};

            if (performance.action_efficiency) fields['Action Efficiency'] = `${Math.round(performance.action_efficiency * 100)}%`;
            if (performance.avg_loop_time) fields['Avg Loop Time'] = `${performance.avg_loop_time.toFixed(2)}s`;
            if (performance.progress_rate) fields['Progress Rate'] = `${Math.round(performance.progress_rate * 100)}%`;

            return fields;
        }

        extractReasoningFields(event) {
            const fields = {};
            const metadata = event.metadata || {};

            if (metadata.outline_step && metadata.outline_total) {
                fields['Outline Progress'] = `${metadata.outline_step}/${metadata.outline_total}`;
            }
            if (metadata.loop_number) fields['Loop Number'] = metadata.loop_number;
            if (metadata.context_size) fields['Context Size'] = metadata.context_size.toLocaleString();
            if (metadata.task_stack_size) fields['Task Stack Size'] = metadata.task_stack_size;

            return fields;
        }

        extractMetadata(event) {
            const fields = {};
            const metadata = event.metadata || {};

            // Show complex data as JSON
            const complexFields = ['tool_args', 'tool_result', 'llm_input', 'llm_output', 'error_details'];

            for (const field of complexFields) {
                if (event[field] && typeof event[field] === 'object') {
                    fields[field.replace(/_/g, ' ').toUpperCase()] = {
                        type: 'json',
                        value: JSON.stringify(event[field], null, 2)
                    };
                }
            }

            return fields;
        }

        // Enhanced helper methods
        // REPLACE the existing getDisplayAction method
        getDisplayAction(eventType, payload) {
            const metadata = payload.metadata || {};
            switch (eventType) {
                case 'reasoning_loop':
                    const step = metadata.outline_step || 0;
                    const total = metadata.outline_total || 0;
                    return step > 0 ? `Reasoning Step ${step}/${total}` : 'Deep Reasoning';
                case 'task_start':
                    return `Starting: ${metadata.description || 'Task'}`;
                case 'task_complete':
                    return `Completed: ${metadata.description || 'Task'}`;
                case 'task_error':
                    return `Failed: ${metadata.description || 'Task'}`;
                case 'tool_call':
                    const status = payload.status || 'running';
                    const toolName = payload.tool_name || 'Unknown Tool';
                    return `${status === 'running' ? 'Calling' : 'Called'} ${toolName}`;

                case 'llm_call':
                    const llmStatus = payload.status || 'running';
                    const model = payload.llm_model || 'LLM';
                    const taskId = payload.task_id || '';

                    // Show more context for LLM calls
                    let displayText = `${llmStatus === 'running' ? '🔄 Calling' : '✅ Called'} ${model}`;
                    if (taskId && taskId !== 'unknown') {
                        displayText += ` (${taskId})`;
                    }
                    return displayText;
                case 'plan_created':
                    return `Plan: ${metadata.plan_name || 'Execution Plan'}`;
                case 'outline_created':
                    return 'Execution Outline Created';
                case 'node_enter':
                    return `Started: ${payload.node_name || 'Processing'}`;
                case 'node_exit':
                    return `Finished: ${payload.node_name || 'Processing'}`;
                case 'node_phase':
                    return `${payload.node_name || 'Node'}: ${payload.node_phase || 'Processing'}`;
                case 'execution_start':
                    return 'Execution Started';
                case 'execution_complete':
                    return 'Execution Complete';
                // ADD: Meta tool events
                case 'meta_tool_call':
                    const metaToolName = metadata.meta_tool_name || payload.tool_name || 'Meta Tool';
                    const metaStatus = payload.status || 'running';
                    return `${metaStatus === 'running' ? 'Using' : 'Used'} ${metaToolName.replace(/_/g, ' ')}`;
                // ADD: Error events
                case 'error':
                    return `Error in ${payload.node_name || 'System'}`;
                default:
                    return eventType.replace(/_/g, ' ').replace(/\b\w/g, l => l.toUpperCase());
            }
        }

        // REPLACE the existing getEventIcon method
        getEventIcon(eventType, status) {
            if (status === 'error' || status === 'failed') return '❌';
            if (status === 'completed') return '✅';

            switch (eventType) {
                case 'reasoning_loop': return '🧠';
                case 'task_start': return '▶️';
                case 'task_complete': return '✅';
                case 'task_error': return '❌';
                case 'tool_call': return '🔧';
                case 'llm_call': return '💭';
                case 'plan_created': return '📋';
                case 'outline_created': return '🗺️';
                case 'node_enter': return '🚀';
                case 'node_exit': return '🏁';
                case 'node_phase': return '⚙️';
                case 'execution_start': return '🎬';
                case 'execution_complete': return '🎉';
                case 'meta_tool_call': return '🛠️';
                case 'error': return '🚨';
                default: return '⚡';
            }
        }

        updateCurrentStatusToIdle() {
            const container = document.getElementById('status-history');
            if (container && container.children.length === 0) {
                container.innerHTML = `
                <div class="progress-item idle-status">
                    <div class="progress-item-header">
                        <div class="progress-icon">💤</div>
                        <div class="progress-title">Ready & Waiting</div>
                        <div class="progress-meta">
                            <span class="progress-status idle">idle</span>
                        </div>
                    </div>
                    <div class="progress-summary">
                        Agent ready for next message • ${new Date().toLocaleTimeString()}
                    </div>
                </div>
            `;
            }
        }

        showTypingIndicator(show) {
            console.log(`💭 ${show ? 'Showing' : 'Hiding'} typing indicator`);
            this.elements.typingIndicator.classList.toggle('active', show);
            if (show) {
                this.elements.typingIndicator.scrollIntoView({ behavior: 'smooth', block: 'end' });
            }
            this.isTyping = show;
        }

        scrollToBottom() {
            if (this.elements.messagesContainer) {
                this.elements.messagesContainer.scrollTop = this.elements.messagesContainer.scrollHeight;
            }
        }

        showApiKeyModal() {
            const storedKey = localStorage.getItem('agent_registry_api_key');
            if (storedKey) {
                this.apiKey = storedKey;
                this.elements.apiKeyModal.style.display = 'none';
                this.connect();
            } else {
                this.elements.apiKeyModal.style.display = 'flex';
            }
        }

        async validateAndStoreApiKey() {
            const apiKey = this.elements.apiKeyInput.value.trim();
            if (!apiKey) {
                this.showError('Please enter an API key');
                return;
            }

            if (!apiKey.startsWith('tbk_')) {
                this.showError('Invalid API key format (should start with tbk_)');
                return;
            }

            this.apiKey = apiKey;
            this.elements.apiKeyModal.style.display = 'none';
            this.connect();
        }

        setupPanelControls() {
            this.elements.sidebarToggle?.addEventListener('click', () => this.togglePanel('sidebar'));
            this.elements.progressToggle?.addEventListener('click', () => this.togglePanel('progress'));
            this.elements.sidebarCollapse?.addEventListener('click', () => this.togglePanel('sidebar'));
            this.elements.progressCollapse?.addEventListener('click', () => this.togglePanel('progress'));

            const mobileTabs = document.querySelectorAll('.mobile-tab');
            if (mobileTabs.length > 0) {
                mobileTabs.forEach(tab => {
                    tab.addEventListener('click', () => this.switchMobileTab(tab.dataset.tab));
                });
            }

            this.setupResponsiveHandlers();
        }

        togglePanel(panel) {
            this.panelStates[panel] = !this.panelStates[panel];
            this.updatePanelStates();
        }

        updatePanelStates() {
            const { sidebar, progress } = this.panelStates;

            if (this.elements.mainContainer) {
                this.elements.mainContainer.classList.remove('sidebar-collapsed', 'progress-collapsed', 'both-collapsed');
                if (!sidebar && !progress) {
                    this.elements.mainContainer.classList.add('both-collapsed');
                } else if (!sidebar) {
                    this.elements.mainContainer.classList.add('sidebar-collapsed');
                } else if (!progress) {
                    this.elements.mainContainer.classList.add('progress-collapsed');
                }
            }

            if (this.elements.sidebar) this.elements.sidebar.classList.toggle('collapsed', !sidebar);
            if (this.elements.progressPanel) this.elements.progressPanel.classList.toggle('collapsed', !progress);

            if (this.elements.sidebarToggle) {
                this.elements.sidebarToggle.classList.toggle('active', sidebar);
                this.elements.sidebarToggle.textContent = sidebar ? '📋 Agents' : '📋';
            }

            if (this.elements.progressToggle) {
                this.elements.progressToggle.classList.toggle('active', progress);
                this.elements.progressToggle.textContent = progress ? '📊 Progress' : '📊';
            }

            if (this.elements.sidebarCollapse) this.elements.sidebarCollapse.textContent = sidebar ? '◀' : '▶';
            if (this.elements.progressCollapse) this.elements.progressCollapse.textContent = progress ? '▶' : '◀';

            if (this.elements.mainContainer) this.elements.mainContainer.offsetHeight;
        }

        handleWindowResize() {
            const chatArea = document.querySelector('.chat-area');
            const mainContainer = this.elements.mainContainer;

            if (chatArea && mainContainer) {
                const currentDisplay = mainContainer.style.display;
                mainContainer.style.display = 'none';
                mainContainer.offsetHeight;
                mainContainer.style.display = currentDisplay || '';
            }
        }

        switchMobileTab(tab) {
            this.panelStates.mobile = tab;

            const mobileTabs = document.querySelectorAll('.mobile-tab');
            if (mobileTabs.length > 0) {
                mobileTabs.forEach(t => t.classList.toggle('active', t.dataset.tab === tab));
            }

            const sidebarEl = document.querySelector('.sidebar');
            const chatAreaEl = document.querySelector('.chat-area');
            const progressPanelEl = document.querySelector('.progress-panel');

            if (sidebarEl) sidebarEl.style.display = tab === 'agents' ? 'flex' : 'none';
            if (chatAreaEl) chatAreaEl.style.display = tab === 'chat' ? 'flex' : 'none';
            if (progressPanelEl) progressPanelEl.style.display = tab === 'progress' ? 'flex' : 'none';
        }

        setupResponsiveHandlers() {
            const mediaQuery = window.matchMedia('(max-width: 768px)');
            const handleResponsive = (e) => {
                if (e.matches) {
                    this.switchMobileTab(this.panelStates.mobile);
                } else {
                    const panels = document.querySelectorAll('.sidebar, .chat-area, .progress-panel');
                    panels.forEach(panel => { if (panel) panel.style.display = ''; });
                }
            };

            if (mediaQuery.addEventListener) {
                mediaQuery.addEventListener('change', handleResponsive);
            } else {
                mediaQuery.addListener(handleResponsive);
            }
            handleResponsive(mediaQuery);
        }

        setupEventListeners() {
            this.elements.apiKeySubmit?.addEventListener('click', () => this.validateAndStoreApiKey());
            window.addEventListener('resize', () => this.handleWindowResize());
            this.elements.apiKeyInput?.addEventListener('keypress', (e) => {
                if (e.key === 'Enter') this.validateAndStoreApiKey();
            });
            this.elements.sendButton.addEventListener('click', () => this.sendMessage());
            this.elements.messageInput.addEventListener('keypress', (e) => {
                if (e.key === 'Enter' && !e.shiftKey && this.currentAgent) {
                    e.preventDefault();
                    this.sendMessage();
                }
            });

            document.addEventListener('visibilitychange', () => {
                if (!document.hidden && (!this.ws || this.ws.readyState === WebSocket.CLOSED)) {
                    this.connect();
                }
            });
        }

        connect() {
            if (this.ws && this.ws.readyState === WebSocket.OPEN) return;

            this.updateConnectionStatus('connecting', 'Connecting...');

            try {
                const isLocal = window.location.hostname === 'localhost' || window.location.hostname === '127.0.0.1';
                const wsProtocol = isLocal ? 'ws' : 'wss';
                const wsUrl = `${wsProtocol}://${window.location.host}/ws/registry/ui_connect`;
                this.ws = new WebSocket(wsUrl);

                // Heartbeat mechanism
                this.heartbeatInterval = setInterval(() => {
                    if (this.ws && this.ws.readyState === WebSocket.OPEN) {
                        this.ws.send(JSON.stringify({event: 'ping', data: {}}));
                    }
                }, 25000); // Send ping every 25 seconds

                this.ws.onopen = () => {
                    this.isConnected = true;
                    this.reconnectAttempts = 0;
                    this.updateConnectionStatus('connected', 'Connected');
                    console.log('Connected to Registry Server');
                };

                this.ws.onmessage = (event) => {
                    try {
                        const data = JSON.parse(event.data);
                        if (data.event === 'pong') {
                            // Handle pong response
                            return;
                        }
                        this.handleWebSocketMessage(data);
                    } catch (error) {
                        console.error('Message parse error:', error);
                    }
                };

                this.ws.onclose = (event) => {
                    this.isConnected = false;
                    if (this.heartbeatInterval) {
                        clearInterval(this.heartbeatInterval);
                    }
                    this.updateConnectionStatus('disconnected', 'Disconnected');
                    this.scheduleReconnection();
                };

                this.ws.onerror = (error) => {
                    console.error('WebSocket error:', error);
                    this.updateConnectionStatus('error', 'Connection Error');
                };

            } catch (error) {
                console.error('Connection error:', error);
                this.updateConnectionStatus('error', 'Connection Failed');
                this.scheduleReconnection();
            }
        }

        scheduleReconnection() {
            if (this.reconnectAttempts >= this.maxReconnectAttempts) {
                this.updateConnectionStatus('error', 'Connection Failed (Max attempts reached)');
                return;
            }

            this.reconnectAttempts++;
            const delay = Math.min(this.reconnectDelay * this.reconnectAttempts, 30000);

            this.updateConnectionStatus('connecting', `Reconnecting in ${delay/1000}s (attempt ${this.reconnectAttempts})`);

            setTimeout(() => {
                if (!this.isConnected) this.connect();
            }, delay);
        }

        updateConnectionStatus(status, text) {
            this.elements.connectionStatus.className = `status-indicator ${status}`;
            this.elements.connectionStatus.querySelector('span').textContent = text;
        }

        handleRegistryEvent(data) {
            const event = data.event;
            const payload = data.data || data;

            console.log(`📋 Registry Event: ${event}`, payload);

            switch (event) {
                case 'api_key_validation':
                    if (payload.valid) {
                        console.log('✅ API key validated successfully');
                    } else {
                        this.showError('❌ Invalid API key for this agent');
                        this.currentAgent = null;
                        this.elements.messageInput.disabled = true;
                        this.elements.sendButton.disabled = true;
                    }
                    break;
                case 'agents_list':
                    console.log('📝 Updating agents list:', payload.agents);
                    this.updateAgentsList(payload.agents);
                    break;
                case 'agent_registered':
                    console.log('🆕 Agent registered:', payload);
                    this.addAgent(payload);
                    break;
                case 'error':
                    console.error('❌ WebSocket error:', payload);
                    this.showError(payload.error || payload.message || 'Unknown error');
                    break;
                default:
                    console.log('❓ Unhandled registry event:', event, payload);
            }
        }

        updateAgentsList(agents) {
            this.elements.agentsContainer.innerHTML = '';

            if (!agents || agents.length === 0) {
                this.elements.agentsContainer.innerHTML = '<div style="color: var(--text-muted); font-size: 12px; text-align: center; padding: 20px;">No agents available</div>';
                return;
            }

            agents.forEach(agent => {
                this.agents.set(agent.public_agent_id, agent);
                const agentEl = this.createAgentElement(agent);
                this.elements.agentsContainer.appendChild(agentEl);
            });
        }

        createAgentElement(agent) {
            const div = document.createElement('div');
            div.className = 'agent-item';
            div.dataset.agentId = agent.public_agent_id;

            div.innerHTML = `
            <div class="agent-name">${agent.public_name}</div>
            <div class="agent-description">${agent.description || 'No description'}</div>
            <div class="agent-status ${agent.status}">
                <div class="status-dot"></div>
                <span>${agent.status.toUpperCase()}</span>
            </div>
        `;

            div.addEventListener('click', () => this.selectAgent(agent));
            return div;
        }

        selectAgent(agent) {
            if (!this.apiKey) {
                this.showError('Please set your API key first');
                return;
            }

            this.sendWebSocketMessage({
                event: 'validate_api_key',
                data: { public_agent_id: agent.public_agent_id, api_key: this.apiKey }
            });

            document.querySelectorAll('.agent-item').forEach(el => el.classList.remove('active'));
            document.querySelector(`[data-agent-id="${agent.public_agent_id}"]`)?.classList.add('active');

            this.currentAgent = agent;
            this.elements.chatTitle.textContent = agent.public_name;
            this.elements.chatSubtitle.textContent = agent.description || 'Ready for conversation';

            this.elements.messageInput.disabled = false;
            this.elements.sendButton.disabled = false;

            this.elements.messagesContainer.innerHTML = '';
            this.addMessage('agent', `Hello! I'm ${agent.public_name}. How can I help you?`);

            this.sendWebSocketMessage({
                event: 'subscribe_agent',
                data: { public_agent_id: agent.public_agent_id }
            });

            this.sendWebSocketMessage({
                event: 'get_agent_status',
                data: { public_agent_id: agent.public_agent_id }
            });

            // Reset progress panels
            this.progressHistory = [];
            this.refreshStatusHistory();
            const metricsContainer = document.getElementById('performance-metrics');
            if (metricsContainer) metricsContainer.innerHTML = '<div class="no-data">No metrics available</div>';
            const outlineContainer = document.getElementById('execution-outline');
            if (outlineContainer) outlineContainer.innerHTML = '<div class="no-data">No outline available</div>';
        }

        sendMessage() {
            if (!this.currentAgent || !this.elements.messageInput.value.trim()) return;

            const message = this.elements.messageInput.value.trim();
            this.addMessage('user', message);

            this.sendWebSocketMessage({
                event: 'chat_message',
                data: {
                    public_agent_id: this.currentAgent.public_agent_id,
                    message: message,
                    session_id: this.sessionId,
                    api_key: this.apiKey
                }
            });

            this.elements.messageInput.value = '';

            // Reset progress state
            this.progressHistory = [];
            this.expandedProgressItem = null;
            this.refreshStatusHistory();

            // Failsafe timeout
            setTimeout(() => {
                if (this.currentExecution) {
                    console.log('⏰ Timeout: Hiding typing indicator and resetting execution state');
                    this.showTypingIndicator(false);
                    this.currentExecution = null;
                    this.updateCurrentStatusToIdle();
                    this.showError('Agent response timeout - please try again');
                }
            }, 60000);
        }

        addMessage(sender, content) {
            const messageDiv = document.createElement('div');
            messageDiv.classList.add('message', sender);

            const avatar = document.createElement('div');
            avatar.classList.add('message-avatar');
            avatar.textContent = sender === 'user' ? 'U' : 'AI';

            const contentDiv = document.createElement('div');
            contentDiv.classList.add('message-content');

            if (sender === 'agent' && window.marked) {
                try {
                    contentDiv.innerHTML = marked.parse(content);
                } catch (error) {
                    contentDiv.textContent = content;
                }
            } else {
                contentDiv.textContent = content;
            }

            messageDiv.appendChild(avatar);
            messageDiv.appendChild(contentDiv);

            this.elements.messagesContainer.appendChild(messageDiv);
            this.elements.messagesContainer.scrollTop = this.elements.messagesContainer.scrollHeight;

            if (sender === 'agent') {
                this.showTypingIndicator(false);
                setTimeout(() => {
                    if (this.currentExecution) {
                        this.currentExecution = null;
                        this.updateCurrentStatusToIdle();
                    }
                }, 1000);
            }
        }

        showError(message) {
            const errorDiv = document.createElement('div');
            errorDiv.className = 'error-message';
            errorDiv.textContent = message;

            document.body.appendChild(errorDiv);
            setTimeout(() => {
                if (errorDiv.parentNode) {
                    errorDiv.parentNode.removeChild(errorDiv);
                }
            }, 5000);
        }

        sendWebSocketMessage(data) {
            if (this.ws && this.ws.readyState === WebSocket.OPEN) {
                this.ws.send(JSON.stringify(data));
            } else {
                console.warn('WebSocket not connected, cannot send message');
            }
        }

    }

    // Initialize UI when DOM is ready
    if (!window.TB) {
        document.addEventListener('DOMContentLoaded', () => {
            window.agentUI = new AgentRegistryUI();
        });
    } else {
        TB.once(() => {
            window.agentUI = new AgentRegistryUI();
        });
    }
</script>
</body>
</html>"""

registry

client

RegistryClient

Manages the client-side connection to the Registry Server with robust reconnection and long-running support.

Source code in toolboxv2/mods/registry/client.py

class RegistryClient:
    """Manages the client-side connection to the Registry Server with robust reconnection and long-running support."""

    def __init__(self, app: App):
        self.app = app

        # WebSocket connection
        self.ws: ws_client.WebSocketClientProtocol | None = None
        self.server_url: str | None = None

        # Task management
        self.connection_task: asyncio.Task | None = None
        self.ping_task: asyncio.Task | None = None
        self.message_handler_tasks: set[asyncio.Task] = set()
        self.progress_processor_task: asyncio.Task | None = None

        # Connection state
        self.is_connected = False
        self.should_reconnect = True
        self.reconnect_in_progress = False
        self.reconnect_attempts = 0
        self.max_reconnect_attempts = 10

        # Agent management
        self.local_agents: dict[str, Any] = {}
        self.registered_info: dict[str, AgentRegistered] = {}
        self.running_executions: dict[str, asyncio.Task] = {}
        self.persistent_callbacks: dict[str, Callable] = {}

        # Progress streaming (NO BATCHING - immediate streaming)
        self.progress_queues: dict[str, asyncio.Queue] = {}
        self.active_streams: set[str] = set()

        # Event handling
        self.custom_event_handlers: dict[str, Callable[[dict], Awaitable[None]]] = {}
        self.pending_registrations: dict[str, asyncio.Future] = {}
        self.registration_counter = 0

    # Utility Methods
    async def get_connection_status(self) -> dict[str, Any]:
        """Get detailed connection status information."""
        try:
            connection_status = {
                "is_connected": self.is_connected,
                "server_url": self.server_url,
                "reconnect_attempts": self.reconnect_attempts,
                "max_reconnect_attempts": self.max_reconnect_attempts,
                "should_reconnect": self.should_reconnect,
                "reconnect_in_progress": self.reconnect_in_progress,
                "websocket_state": None,
                "websocket_open": False,
                "tasks": {
                    "connection_task_running": self.connection_task and not self.connection_task.done(),
                    "ping_task_running": self.ping_task and not self.ping_task.done(),
                },
                "registered_agents_count": len(self.local_agents),
                "running_executions_count": len(self.running_executions),
                "pending_registrations_count": len(self.pending_registrations),
                "persistent_callbacks_count": len(self.persistent_callbacks),
                "last_ping_time": getattr(self, 'last_ping_time', None),
                "connection_uptime": None,
                "connection_established_at": getattr(self, 'connection_established_at', None),
            }

            # WebSocket specific status
            if self.ws:
                connection_status.update({
                    "websocket_state": str(self.ws.state.name) if hasattr(self.ws.state, 'name') else str(
                        self.ws.state),
                    "websocket_open": self.ws.open,
                    "websocket_closed": self.ws.closed,
                })

            # Calculate uptime
            if hasattr(self, 'connection_established_at') and self.connection_established_at:
                connection_status[
                    "connection_uptime"] = asyncio.get_event_loop().time() - self.connection_established_at

            return connection_status

        except Exception as e:
            self.app.print(f"Error getting connection status: {e}")
            return {
                "error": str(e),
                "is_connected": False,
                "server_url": self.server_url,
            }

    async def get_registered_agents(self) -> dict[str, AgentRegistered]:
        """Get all registered agents information."""
        try:
            agents_info = {}

            for agent_id, reg_info in self.registered_info.items():
                # Get agent instance if available
                agent_instance = self.local_agents.get(agent_id)

                # Create enhanced agent info
                agent_data = {
                    "registration_info": reg_info,
                    "agent_available": agent_instance is not None,
                    "agent_type": type(agent_instance).__name__ if agent_instance else "Unknown",
                    "has_progress_callback": hasattr(agent_instance, 'progress_callback') if agent_instance else False,
                    "supports_progress_callback": hasattr(agent_instance,
                                                          'set_progress_callback') if agent_instance else False,
                    "is_persistent_callback_active": agent_id in self.persistent_callbacks,
                    "registration_timestamp": getattr(reg_info, 'registration_timestamp', None),
                }

                # Add agent capabilities if available
                if agent_instance and hasattr(agent_instance, 'get_capabilities'):
                    try:
                        agent_data["capabilities"] = await agent_instance.get_capabilities()
                    except Exception as e:
                        agent_data["capabilities_error"] = str(e)

                agents_info[agent_id] = agent_data

            return agents_info

        except Exception as e:
            self.app.print(f"Error getting registered agents: {e}")
            return {}

    async def get_running_executions(self) -> dict[str, dict[str, Any]]:
        """Get information about currently running executions."""
        try:
            executions_info = {}

            for request_id, execution_task in self.running_executions.items():
                execution_info = {
                    "request_id": request_id,
                    "task_done": execution_task.done(),
                    "task_cancelled": execution_task.cancelled(),
                    "start_time": getattr(execution_task, 'start_time', None),
                    "running_time": None,
                    "task_exception": None,
                    "task_result": None,
                }

                # Calculate running time
                if hasattr(execution_task, 'start_time') and execution_task.start_time:
                    execution_info["running_time"] = asyncio.get_event_loop().time() - execution_task.start_time

                # Get task status details
                if execution_task.done():
                    try:
                        if execution_task.exception():
                            execution_info["task_exception"] = str(execution_task.exception())
                        else:
                            execution_info["task_result"] = "completed_successfully"
                    except Exception as e:
                        execution_info["task_status_error"] = str(e)

                executions_info[request_id] = execution_info

            return executions_info

        except Exception as e:
            self.app.print(f"Error getting running executions: {e}")
            return {}

    async def cancel_execution(self, request_id: str) -> bool:
        """Cancel a running execution."""
        try:
            if request_id not in self.running_executions:
                self.app.print(f"❌ Execution {request_id} not found")
                return False

            execution_task = self.running_executions[request_id]

            if execution_task.done():
                self.app.print(f"⚠️  Execution {request_id} already completed")
                return True

            # Cancel the task
            execution_task.cancel()

            try:
                # Wait a moment for graceful cancellation
                await asyncio.wait_for(execution_task, timeout=5.0)
            except asyncio.CancelledError:
                self.app.print(f"✅ Execution {request_id} cancelled successfully")
            except asyncio.TimeoutError:
                self.app.print(f"⚠️  Execution {request_id} cancellation timeout - may still be running")
            except Exception as e:
                self.app.print(f"⚠️  Execution {request_id} cancellation resulted in exception: {e}")

            # Send cancellation notice to server
            try:
                if self.is_connected and self.ws and self.ws.open:
                    cancellation_event = ProgressEvent(
                        event_type="execution_cancelled",
                        node_name="RegistryClient",
                        success=False,
                        metadata={
                            "request_id": request_id,
                            "cancellation_reason": "client_requested",
                            "timestamp": asyncio.get_event_loop().time()
                        }
                    )

                    cancellation_message = ExecutionResult(
                        request_id=request_id,
                        payload=cancellation_event.to_dict(),
                        is_final=True
                    )

                    await self._send_message('execution_result', cancellation_message.model_dump())

            except Exception as e:
                self.app.print(f"Failed to send cancellation notice to server: {e}")

            # Cleanup
            self.running_executions.pop(request_id, None)

            return True

        except Exception as e:
            self.app.print(f"Error cancelling execution {request_id}: {e}")
            return False

    async def health_check(self) -> bool:
        """Perform a health check of the connection."""
        try:
            # Basic connection checks
            if not self.is_connected:
                self.app.print("🔍 Health check: Not connected")
                return False

            if not self.ws or not self.ws.open:
                self.app.print("🔍 Health check: WebSocket not open")
                return False

            # Ping test
            try:
                pong_waiter = await self.ws.ping()
                await asyncio.wait_for(pong_waiter, timeout=10.0)

                # Update last ping time
                self.last_ping_time = asyncio.get_event_loop().time()

                # Test message sending
                test_message = WsMessage(
                    event='health_check',
                    data={
                        "timestamp": self.last_ping_time,
                        "client_id": getattr(self, 'client_id', 'unknown'),
                        "registered_agents": list(self.local_agents.keys()),
                        "running_executions": list(self.running_executions.keys())
                    }
                )

                await self.ws.send(test_message.model_dump_json())

                self.app.print("✅ Health check: Connection healthy")
                return True

            except asyncio.TimeoutError:
                self.app.print("❌ Health check: Ping timeout")
                return False
            except Exception as ping_error:
                self.app.print(f"❌ Health check: Ping failed - {ping_error}")
                return False

        except Exception as e:
            self.app.print(f"❌ Health check: Error - {e}")
            return False

    async def get_diagnostics(self) -> dict[str, Any]:
        """Get comprehensive diagnostic information."""
        try:
            diagnostics = {
                "connection_status": await self.get_connection_status(),
                "registered_agents": await self.get_registered_agents(),
                "running_executions": await self.get_running_executions(),
                "health_status": await self.health_check(),
                "system_info": {
                    "python_version": sys.version,
                    "asyncio_running": True,
                    "event_loop": str(asyncio.get_running_loop()),
                    "thread_name": threading.current_thread().name,
                },
                "performance_metrics": {
                    "total_messages_sent": getattr(self, 'total_messages_sent', 0),
                    "total_messages_received": getattr(self, 'total_messages_received', 0),
                    "total_reconnections": self.reconnect_attempts,
                    "total_registrations": len(self.registered_info),
                    "memory_usage": self._get_memory_usage(),
                },
                "error_log": getattr(self, 'recent_errors', []),
            }

            return diagnostics

        except Exception as e:
            return {
                "diagnostics_error": str(e),
                "timestamp": asyncio.get_event_loop().time()
            }

    def _get_memory_usage(self) -> dict[str, Any]:
        """Get memory usage information."""
        try:
            import psutil
            import os

            process = psutil.Process(os.getpid())
            memory_info = process.memory_info()

            return {
                "rss": memory_info.rss,
                "vms": memory_info.vms,
                "percent": process.memory_percent(),
                "available": psutil.virtual_memory().available,
            }
        except ImportError:
            return {"error": "psutil not available"}
        except Exception as e:
            return {"error": str(e)}

    async def cleanup_completed_executions(self):
        """Clean up completed execution tasks."""
        try:
            completed_tasks = []

            for request_id, task in self.running_executions.items():
                if task.done():
                    completed_tasks.append(request_id)

            for request_id in completed_tasks:
                self.running_executions.pop(request_id, None)
                self.app.print(f"🧹 Cleaned up completed execution: {request_id}")

            return len(completed_tasks)

        except Exception as e:
            self.app.print(f"Error during cleanup: {e}")
            return 0

    async def connect(self, server_url: str, timeout: float = 30.0):
        """Connect and start all background tasks."""
        if not ws_client:
            self.app.print("Websockets library not installed. Please run 'pip install websockets'")
            return False

        if self.ws and self.ws.open:
            self.app.print("Already connected to the registry server.")
            return True

        self.server_url = server_url
        self.should_reconnect = True
        self.reconnect_in_progress = False

        try:
            self.app.print(f"Connecting to Registry Server at {server_url}...")
            self.ws = await asyncio.wait_for(
                ws_client.connect(server_url),
                timeout=timeout
            )

            self.is_connected = True
            self.reconnect_attempts = 0

            # Start all background tasks
            await self._start_all_background_tasks()

            self.app.print(f"✅ Successfully connected and started all tasks")
            return True

        except asyncio.TimeoutError:
            self.app.print(f"❌ Connection timeout after {timeout}s")
            return False
        except Exception as e:
            self.app.print(f"❌ Connection failed: {e}")
            return False

    async def _start_all_background_tasks(self):
        """Start all background tasks needed for operation."""
        # Start connection listener
        self.connection_task = asyncio.create_task(self._listen())

        # Start ping task
        self.ping_task = asyncio.create_task(self._ping_loop())

        self.app.print("🚀 All background tasks started")
    async def _start_ping_task(self):
        """Start the ping/heartbeat task in the background."""
        if self.ping_task and not self.ping_task.done():
            return  # Already running

        self.ping_task = asyncio.create_task(self._ping_loop())

    async def _ping_loop(self):
        """Dedicated ping task that never blocks and has highest priority."""
        ping_interval = 20  # Less aggressive than server's 5s interval
        consecutive_failures = 0
        max_failures = 2

        while self.is_connected and self.should_reconnect:
            try:
                await asyncio.sleep(ping_interval)

                # Double-check connection state
                if not self.ws or not self.ws.open or self.ws.closed:
                    self.app.print("Ping task detected closed connection")
                    break

                try:
                    # Send ping with short timeout
                    pong_waiter = await self.ws.ping()
                    await asyncio.wait_for(pong_waiter, timeout=8.0)  # Less than server's 10s timeout

                    consecutive_failures = 0
                    self.app.print("📡 Heartbeat successful")

                except asyncio.TimeoutError:
                    consecutive_failures += 1
                    self.app.print(f"⚠️ Ping timeout ({consecutive_failures}/{max_failures})")

                    if consecutive_failures >= max_failures:
                        self.app.print("❌ Multiple ping timeouts - connection dead")
                        break

                except Exception as ping_error:
                    consecutive_failures += 1
                    self.app.print(f"❌ Ping error ({consecutive_failures}/{max_failures}): {ping_error}")

                    if consecutive_failures >= max_failures:
                        break

            except Exception as e:
                self.app.print(f"Ping loop error: {e}")
                break

        self.app.print("Ping task stopped")
        # Trigger reconnect if we should still be connected
        if self.should_reconnect and self.is_connected:
            asyncio.create_task(self._trigger_reconnect())

    async def _trigger_reconnect(self):
        """Trigger a reconnection attempt."""
        if self.reconnect_in_progress:
            return

        self.reconnect_in_progress = True
        self.is_connected = False

        try:
            if self.ws:
                with contextlib.suppress(Exception):
                    await self.ws.close()
                self.ws = None

            # Stop current tasks
            if self.connection_task and not self.connection_task.done():
                self.connection_task.cancel()
            if self.ping_task and not self.ping_task.done():
                self.ping_task.cancel()

            self.app.print("🔄 Attempting to reconnect...")
            await self._reconnect_with_backoff()

        finally:
            self.reconnect_in_progress = False

    async def _reconnect_with_backoff(self):
        """Reconnect with exponential backoff."""
        max_attempts = 10
        base_delay = 2
        max_delay = 300  # 5 minutes max

        for attempt in range(max_attempts):
            if not self.should_reconnect:
                break

            delay = min(base_delay * (2 ** attempt), max_delay)
            self.app.print(f"🔄 Reconnect attempt {attempt + 1}/{max_attempts} in {delay}s...")

            await asyncio.sleep(delay)

            try:
                if self.server_url:
                    self.ws = await ws_client.connect(self.server_url)
                    self.is_connected = True
                    self.reconnect_attempts = 0

                    # Restart tasks
                    self.connection_task = asyncio.create_task(self._listen())
                    await self._start_ping_task()

                    # Re-register agents
                    await self._reregister_agents()

                    self.app.print("✅ Reconnected successfully!")
                    return

            except Exception as e:
                self.app.print(f"❌ Reconnect attempt {attempt + 1} failed: {e}")

        self.app.print("❌ All reconnection attempts failed")
        self.should_reconnect = False

    async def _reregister_agents(self):
        """Re-register all local agents after reconnection."""
        if not self.registered_info:
            self.app.print("No agents to re-register")
            return

        self.app.print(f"Re-registering {len(self.registered_info)} agents...")

        for agent_id, reg_info in list(self.registered_info.items()):
            try:
                agent_instance = self.local_agents.get(agent_id)
                if not agent_instance:
                    continue

                # Create new registration (server will assign new IDs)
                new_reg_info = await self.register(
                    agent_instance,
                    reg_info.public_name,
                    self.local_agents.get(f"{agent_id}_description", "Re-registered agent")
                )

                if new_reg_info:
                    # Update stored information
                    old_agent_id = agent_id
                    new_agent_id = new_reg_info.public_agent_id

                    # Move agent to new ID
                    self.local_agents[new_agent_id] = self.local_agents.pop(old_agent_id)
                    self.registered_info[new_agent_id] = self.registered_info.pop(old_agent_id)

                    self.app.print(f"✅ Re-registered agent: {reg_info.public_name} (new ID: {new_agent_id})")
                else:
                    self.app.print(f"❌ Failed to re-register agent: {reg_info.public_name}")

            except Exception as e:
                self.app.print(f"Error re-registering agent {reg_info.public_name}: {e}")

        self.app.print("Agent re-registration completed")

    async def _create_persistent_progress_callback(self, request_id: str, agent_id: str):
        """Create progress callback with offline queuing capability."""
        progress_queue = asyncio.Queue(maxsize=100)  # Buffer for offline messages

        async def persistent_progress_callback(event: ProgressEvent):
            try:
                # Add to queue first
                try:
                    progress_queue.put_nowait((event, asyncio.get_event_loop().time()))
                except asyncio.QueueFull:
                    # Remove oldest item and add new one
                    try:
                        progress_queue.get_nowait()
                        progress_queue.put_nowait((event, asyncio.get_event_loop().time()))
                    except asyncio.QueueEmpty:
                        pass

                # Try to send immediately if connected
                if await self._check_connection_health():
                    try:
                        result = ExecutionResult(
                            request_id=request_id,
                            payload=event.to_dict(),
                            is_final=False
                        )
                        success = await self._send_message('execution_result', result.model_dump())
                        if success:
                            # Remove from queue since it was sent successfully
                            try:
                                progress_queue.get_nowait()
                            except asyncio.QueueEmpty:
                                pass
                            return
                    except Exception as e:
                        self.app.print(f"Progress send failed, queued: {e}")

                # If we get here, message is queued for later sending

            except Exception as e:
                self.app.print(f"Progress callback error: {e}")

        # Store queue for later processing
        self.progress_queues[request_id] = progress_queue
        return persistent_progress_callback
    async def _store_progress_callback_state(self, agent_id: str, callback_func):
        """Store progress callback for reconnection scenarios."""
        self.persistent_callbacks[agent_id] = callback_func

    async def _restore_progress_callbacks(self):
        """Restore progress callbacks after reconnection."""
        for agent_id, callback_func in self.persistent_callbacks.items():
            agent = self.local_agents.get(agent_id)
            if agent and hasattr(agent, 'set_progress_callback'):
                agent.set_progress_callback(callback_func)

    def on(self, event_name: str, handler: Callable[[dict], Awaitable[None]]):
        """Register an async callback function to handle a custom event from the server."""
        self.app.print(f"Handler for custom event '{event_name}' registered.")
        self.custom_event_handlers[event_name] = handler

    async def send_custom_event(self, event_name: str, data: dict[str, Any]):
        """Send a custom event with a JSON payload to the server."""
        if not self.is_connected or not self.ws or not self.ws.open:
            self.app.print("Cannot send custom event: Not connected.")
            return

        try:
            message = WsMessage(event=event_name, data=data)
            await self.ws.send(message.model_dump_json())
            self.app.print(f"Sent custom event '{event_name}' to server.")
        except Exception as e:
            self.app.print(f"Failed to send custom event: {e}")
            await self._handle_connection_error()

    async def _listen(self):
        """Robust message listening loop with immediate connection loss detection."""
        self.app.print("Registry client is now listening for incoming requests...")

        try:
            while self.is_connected and self.ws and self.ws.open:
                try:
                    # Check connection state before each recv attempt
                    if self.ws.closed:
                        self.app.print("WebSocket is closed - triggering reconnect")
                        break

                    message_raw = await asyncio.wait_for(self.ws.recv(), timeout=5.0)

                    # Handle different message types immediately
                    if isinstance(message_raw, bytes):
                        # Server ping - respond immediately
                        continue

                    # Process text messages
                    try:
                        message = WsMessage.model_validate_json(message_raw)
                        # Handle critical messages immediately, others in background
                        if message.event in ['agent_registered']:
                            await self._handle_message(message)
                        else:
                            # Handle non-critical messages in background to avoid blocking
                            task = asyncio.create_task(self._handle_message(message))
                            self.message_handler_tasks.add(task)
                            # Clean completed tasks
                            self.message_handler_tasks = {t for t in self.message_handler_tasks if not t.done()}

                    except Exception as e:
                        self.app.print(f"Error processing message: {e} | Raw: {message_raw[:200]}")

                except asyncio.TimeoutError:
                    # Normal timeout - check connection health
                    if not self.ws or not self.ws.open or self.ws.closed:
                        self.app.print("Connection health check failed during timeout")
                        break
                    continue

                except ConnectionClosed as e:
                    self.app.print(f"Connection closed by server: {e}")
                    break

                except Exception as e:
                    # Any other WebSocket error means connection is likely dead
                    if "ConnectionClosedError" in str(type(e)) or "IncompleteReadError" in str(type(e)):
                        self.app.print(f"Connection lost: {e}")
                        break
                    else:
                        self.app.print(f"Unexpected error in listen loop: {e}")
                        # Don't break on unexpected errors, but log them
                        await asyncio.sleep(0.1)

        except Exception as e:
            self.app.print(f"Fatal error in listen loop: {e}")
        finally:
            # Always trigger reconnection attempt
            if self.should_reconnect:
                asyncio.create_task(self._trigger_reconnect())

    async def _handle_message(self, message: WsMessage):
        """Handle incoming WebSocket messages with non-blocking execution."""
        try:
            if message.event == 'agent_registered':
                # Handle registration confirmation immediately
                reg_info = AgentRegistered.model_validate(message.data)
                reg_id = None
                for rid, future in self.pending_registrations.items():
                    if not future.done():
                        reg_id = rid
                        break

                if reg_id and reg_id in self.pending_registrations:
                    if not self.pending_registrations[reg_id].done():
                        self.pending_registrations[reg_id].set_result(reg_info)
                else:
                    self.app.print("Received agent_registered but no pending registration found")

            elif message.event == 'run_request':
                # Handle run requests in background - NEVER block here
                run_data = RunRequest.model_validate(message.data)
                asyncio.create_task(self._handle_run_request(run_data))

            elif message.event in self.custom_event_handlers:
                # Handle custom events in background
                self.app.print(f"Received custom event '{message.event}' from server.")
                handler = self.custom_event_handlers[message.event]
                asyncio.create_task(handler(message.data))

            else:
                self.app.print(f"Received unhandled event from server: '{message.event}'")

        except Exception as e:
            self.app.print(f"Error handling message: {e}")
            # Don't let message handling errors kill the connection

    async def register(self, agent_instance: Any, public_name: str, description: str | None = None) -> AgentRegistered | None:
        """Register an agent with the server."""
        if not self.is_connected or not self.ws:
            self.app.print("Not connected. Cannot register agent.")
            return None

        try:
            # Create registration request
            registration = AgentRegistration(public_name=public_name, description=description)
            message = WsMessage(event='register', data=registration.model_dump())

            # Create future for registration response
            reg_id = f"reg_{self.registration_counter}"
            self.registration_counter += 1
            self.pending_registrations[reg_id] = asyncio.Future()

            # Send registration request
            await self.ws.send(message.model_dump_json())
            self.app.print(f"Sent registration request for agent '{public_name}'")

            # Wait for registration confirmation
            try:
                reg_info = await asyncio.wait_for(self.pending_registrations[reg_id], timeout=30.0)

                # Store agent and registration info
                self.local_agents[reg_info.public_agent_id] = agent_instance
                self.registered_info[reg_info.public_agent_id] = reg_info

                self.app.print(f"Agent '{public_name}' registered successfully.")
                self.app.print(f"  Public URL: {reg_info.public_url}")
                self.app.print(f"  API Key: {reg_info.public_api_key}")

                return reg_info

            except TimeoutError:
                self.app.print("Timeout waiting for registration confirmation.")
                return None

        except Exception as e:
            self.app.print(f"Error during registration: {e}")
            return None
        finally:
            # Cleanup pending registration
            self.pending_registrations.pop(reg_id, None)

    async def _handle_run_request(self, run_request: RunRequest):
        """Handle run request - start agent in completely separate task."""
        agent_id = run_request.public_agent_id
        agent = self.local_agents.get(agent_id)

        if not agent:
            await self._stream_error(run_request.request_id, f"Agent with ID {agent_id} not found")
            return

        # Start agent execution in separate task - NEVER await here
        execution_task = asyncio.create_task(
            self._execute_agent_with_monitoring(agent, run_request)
        )

        # Store task but don't wait for it
        self.running_executions[run_request.request_id] = execution_task

        self.app.print(f"🚀 Agent execution started in background: {run_request.request_id}")
        # This method returns immediately - agent runs in background
    async def _execute_agent_with_monitoring(self, agent: Any, run_request: RunRequest):
        """Execute agent in completely separate task - never blocks main connection."""
        request_id = run_request.request_id
        agent_id = run_request.public_agent_id

        try:
            # Create progress streaming callback
            progress_callback = await self._create_streaming_progress_callback(request_id, agent_id)

            # Store original callback
            original_callback = getattr(agent, 'progress_callback', None)

            # Set streaming progress callback
            if hasattr(agent, 'set_progress_callback'):
                agent.set_progress_callback(progress_callback)
            elif hasattr(agent, 'progress_callback'):
                agent.progress_callback = progress_callback

            # Store for reconnection scenarios
            self.persistent_callbacks[agent_id] = progress_callback
            self.active_streams.add(request_id)

            self.app.print(f"🚀 Starting agent execution in separate task: {request_id}")

            # EXECUTE THE AGENT - this can run for hours/days
            final_result = await agent.a_run(
                query=run_request.query,
                session_id=run_request.session_id,
                **run_request.kwargs
            )

            # Send final result
            await self._stream_final_result(request_id, final_result, agent_id, run_request.session_id)

            self.app.print(f"✅ Agent execution completed: {request_id}")

        except Exception as e:
            self.app.print(f"❌ Agent execution failed: {e}")
            await self._stream_error(request_id, str(e))
            import traceback
            traceback.print_exc()

        finally:
            # Cleanup
            await self.running_executions.pop(request_id, None)
            self.persistent_callbacks.pop(agent_id, None)
            self.active_streams.discard(request_id)

            # Close progress queue
            if request_id in self.progress_queues:
                queue = self.progress_queues.pop(request_id)
                # Signal queue processor to stop for this request
                try:
                    await queue.put(None)  # Sentinel value
                except:
                    pass

            # Restore original callback
            try:
                if hasattr(agent, 'set_progress_callback'):
                    agent.set_progress_callback(original_callback)
                elif hasattr(agent, 'progress_callback'):
                    agent.progress_callback = original_callback
            except Exception as cleanup_error:
                self.app.print(f"Warning: Callback cleanup failed: {cleanup_error}")

    async def _stream_final_result(self, request_id: str, final_result: Any, agent_id: str, session_id: str):
        """Stream final result immediately."""
        final_event = ProgressEvent(
            event_type="execution_complete",
            node_name="RegistryClient",
            success=True,
            metadata={
                "result": final_result,
                "agent_id": agent_id,
                "session_id": session_id
            }
        )

        final_message = ExecutionResult(
            request_id=request_id,
            payload=final_event.to_dict(),
            is_final=True
        )

        # Stream final result with high priority
        max_attempts = 10
        for attempt in range(max_attempts):
            try:
                if await self._check_connection_health():
                    success = await self._send_message('execution_result', final_message.model_dump())
                    if success:
                        self.app.print(f"✅ Final result streamed successfully")
                        return

                await asyncio.sleep(1.0 * (attempt + 1))  # Longer delays for final result

            except Exception as e:
                self.app.print(f"Final result stream attempt {attempt + 1} failed: {e}")

        self.app.print(f"❌ Failed to stream final result after {max_attempts} attempts")

    async def _stream_error(self, request_id: str, error_message: str):
        """Stream error immediately."""
        error_payload = ExecutionError(request_id=request_id, error=error_message)

        for attempt in range(5):
            try:
                if await self._check_connection_health():
                    success = await self._send_message('execution_error', error_payload.model_dump())
                    if success:
                        return
                await asyncio.sleep(0.5 * (attempt + 1))
            except Exception as e:
                self.app.print(f"Error stream attempt {attempt + 1} failed: {e}")

    async def _create_streaming_progress_callback(self, request_id: str, agent_id: str):
        """Create callback that streams progress immediately as it comes."""
        # Create queue for this specific request
        progress_queue = asyncio.Queue()
        self.progress_queues[request_id] = progress_queue

        # Start dedicated processor for this request
        processor_task = asyncio.create_task(
            self._process_progress_stream(request_id, progress_queue)
        )

        async def streaming_progress_callback(event: ProgressEvent):
            """Stream progress immediately - no batching, no delays."""
            try:
                if request_id in self.active_streams:
                    # Put in queue for immediate processing
                    await progress_queue.put(event)
            except Exception as e:
                self.app.print(f"Progress streaming error: {e}")

        return streaming_progress_callback

    async def _process_progress_stream(self, request_id: str, progress_queue: asyncio.Queue):
        """Process progress stream in real-time - separate task per request."""
        self.app.print(f"📡 Started progress streaming for request: {request_id}")

        while request_id in self.active_streams:
            try:
                # Get next progress event (blocking)
                event = await progress_queue.get()

                # Sentinel value to stop
                if event is None:
                    break

                # Stream immediately - no batching
                await self._stream_progress_immediately(request_id, event)

            except Exception as e:
                self.app.print(f"Progress stream processing error: {e}")
                await asyncio.sleep(0.1)  # Brief pause on error

        self.app.print(f"📡 Stopped progress streaming for request: {request_id}")

    async def _stream_progress_immediately(self, request_id: str, event: ProgressEvent):
        """Stream single progress event immediately."""
        max_attempts = 3

        for attempt in range(max_attempts):
            try:
                if await self._check_connection_health():
                    result = ExecutionResult(
                        request_id=request_id,
                        payload=event.to_dict(),
                        is_final=False
                    )

                    success = await self._send_message('execution_result', result.model_dump())
                    if success:
                        return  # Successfully streamed

                # Connection unhealthy - brief wait before retry
                await asyncio.sleep(0.2 * (attempt + 1))

            except Exception as e:
                self.app.print(f"Stream attempt {attempt + 1} failed: {e}")
                if attempt < max_attempts - 1:
                    await asyncio.sleep(0.2 * (attempt + 1))

        # All attempts failed - but don't crash, just log
        self.app.print(f"⚠️ Failed to stream progress after {max_attempts} attempts")


    async def send_ui_progress(self, progress_data: dict[str, Any], retry_count: int = 3):
        """Enhanced UI progress sender with retry logic."""
        if not self.is_connected or not self.ws or not self.ws.open:
            self.app.print("Registry client WebSocket not connected - queuing progress update")
            # Could implement a queue here for offline progress updates
            return False

        for attempt in range(retry_count):
            try:
                # Structure progress message for registry server
                ui_message = {
                    "timestamp": progress_data.get('timestamp', asyncio.get_event_loop().time()),
                    "agent_id": progress_data.get('agent_id', 'unknown'),
                    "event_type": progress_data.get('event_type', 'unknown'),
                    "status": progress_data.get('status', 'processing'),
                    "agent_name": progress_data.get('agent_name', 'Unknown'),
                    "node_name": progress_data.get('node_name', 'Unknown'),
                    "session_id": progress_data.get('session_id'),
                    "metadata": progress_data.get('metadata', {}),

                    # Enhanced progress data for UI panels
                    "outline_progress": progress_data.get('progress_data', {}).get('outline', {}),
                    "activity_info": progress_data.get('progress_data', {}).get('activity', {}),
                    "meta_tool_info": progress_data.get('progress_data', {}).get('meta_tool', {}),
                    "system_status": progress_data.get('progress_data', {}).get('system', {}),
                    "graph_info": progress_data.get('progress_data', {}).get('graph', {}),

                    # UI flags for selective updates
                    "ui_flags": progress_data.get('ui_flags', {}),

                    # Performance metrics
                    "performance": progress_data.get('performance', {}),

                    # Message metadata
                    "message_id": f"msg_{asyncio.get_event_loop().time()}_{attempt}",
                    "retry_count": attempt
                }

                # Send as WsMessage
                message = WsMessage(event='ui_progress_update', data=ui_message)
                await self.ws.send(message.model_dump_json())

                # Success - break retry loop
                self.app.print(
                    f"📤 Sent UI progress: {progress_data.get('event_type')} | {progress_data.get('status')} (attempt {attempt + 1})")
                return True

            except Exception as e:
                self.app.print(f"Failed to send UI progress (attempt {attempt + 1}/{retry_count}): {e}")
                if attempt < retry_count - 1:
                    await asyncio.sleep(0.5 * (attempt + 1))  # Exponential backoff
                else:
                    await self._handle_connection_error()
                    return False

        return False

    async def send_agent_status(self, agent_id: str, status: str, details: dict[str, Any] = None):
        """Send agent status updates."""
        if not self.is_connected or not self.ws or not self.ws.open:
            return

        try:
            status_message = {
                "agent_id": agent_id,
                "status": status,
                "details": details or {},
                "timestamp": asyncio.get_event_loop().time(),
                "capabilities": ["chat", "progress_tracking", "outline_visualization", "meta_tool_monitoring"]
            }

            message = WsMessage(event='agent_status_update', data=status_message)
            await self.ws.send(message.model_dump_json())

        except Exception as e:
            self.app.print(f"Failed to send agent status: {e}")
            await self._handle_connection_error()

    async def _send_error(self, request_id: str, error_message: str):
        """Send error message to server."""
        error_payload = ExecutionError(request_id=request_id, error=error_message)
        await self._send_message('execution_error', error_payload.model_dump())

    async def _check_connection_health(self) -> bool:
        """Check if the WebSocket connection is actually healthy."""
        if not self.ws:
            return False

        try:
            # Check basic connection state
            if self.ws.closed or not self.ws.open:
                return False

            # Try a quick ping to verify connectivity
            pong_waiter = await self.ws.ping()
            await asyncio.wait_for(pong_waiter, timeout=3.0)
            return True

        except Exception as e:
            self.app.print(f"Connection health check failed: {e}")
            return False

    async def _send_message(self, event: str, data: dict, max_retries: int = 3):
        """Enhanced message sending with connection health verification."""
        for attempt in range(max_retries):
            # Check connection health before attempting to send
            if not await self._check_connection_health():
                self.app.print(f"Connection unhealthy for message '{event}' (attempt {attempt + 1})")

                if attempt < max_retries - 1:
                    await asyncio.sleep(0.5 * (attempt + 1))
                    continue
                else:
                    self.app.print(f"Cannot send message '{event}': Connection permanently failed")
                    asyncio.create_task(self._trigger_reconnect())
                    return False

            try:
                message = WsMessage(event=event, data=data)
                await self.ws.send(message.model_dump_json())
                return True

            except Exception as e:
                self.app.print(f"Send attempt {attempt + 1} failed for '{event}': {e}")

                # Check if this is a connection-related error
                error_str = str(e).lower()
                if any(err in error_str for err in ['connectionclosed', 'incomplete', 'connection', 'closed']):
                    self.app.print("Connection error detected - triggering reconnect")
                    asyncio.create_task(self._trigger_reconnect())
                    return False

                if attempt < max_retries - 1:
                    await asyncio.sleep(0.5 * (attempt + 1))

        return False
    async def _send_final_result_with_retry(self, request_id: str, final_result: Any, agent_id: str, session_id: str):
        """Send final result with robust retry logic."""
        final_event = ProgressEvent(
            event_type="execution_complete",
            node_name="RegistryClient",
            success=True,
            metadata={
                "result": final_result,
                "agent_id": agent_id,
                "session_id": session_id
            }
        )

        final_message = ExecutionResult(
            request_id=request_id,
            payload=final_event.to_dict(),
            is_final=True
        )

        max_retries = 10
        base_delay = 2

        for attempt in range(max_retries):
            try:
                if not self.is_connected or not self.ws or not self.ws.open:
                    self.app.print(f"⚠️  Connection lost - waiting for reconnection (attempt {attempt + 1})")
                    await asyncio.sleep(base_delay * (attempt + 1))
                    continue

                await self._send_message('execution_result', final_message.model_dump())
                self.app.print(f"✅ Final result sent successfully on attempt {attempt + 1}")
                return

            except Exception as e:
                delay = base_delay * (2 ** attempt)
                self.app.print(f"❌ Failed to send final result (attempt {attempt + 1}): {e}")
                if attempt < max_retries - 1:
                    await asyncio.sleep(delay)

        self.app.print(f"❌ Failed to send final result after {max_retries} attempts")

    async def _send_error_with_retry(self, request_id: str, error_message: str):
        """Send error message with retry logic."""
        max_retries = 5

        for attempt in range(max_retries):
            try:
                if self.is_connected and self.ws and self.ws.open:
                    await self._send_error(request_id, error_message)
                    return
                else:
                    await asyncio.sleep(2 * (attempt + 1))
            except Exception as e:
                self.app.print(f"Error sending error message (attempt {attempt + 1}): {e}")
                if attempt < max_retries - 1:
                    await asyncio.sleep(2 * (attempt + 1))

    async def _handle_connection_error(self):
        """Handle connection errors and cleanup."""
        self.is_connected = False
        if self.ws:
            with contextlib.suppress(builtins.BaseException):
                await self.ws.close()
            self.ws = None

    async def disconnect(self):
        """Enhanced disconnect with complete task cleanup."""
        self.app.print("Initiating clean shutdown...")
        self.is_connected = False
        self.should_reconnect = False

        # Cancel all background tasks
        tasks_to_cancel = []

        if self.connection_task and not self.connection_task.done():
            tasks_to_cancel.append(self.connection_task)

        if self.ping_task and not self.ping_task.done():
            tasks_to_cancel.append(self.ping_task)

        # Cancel message handler tasks
        for task in list(self.message_handler_tasks):
            if not task.done():
                tasks_to_cancel.append(task)

        # Cancel running executions
        for task in list(self.running_executions.values()):
            if not task.done():
                tasks_to_cancel.append(task)

        if tasks_to_cancel:
            self.app.print(f"Cancelling {len(tasks_to_cancel)} background tasks...")
            for task in tasks_to_cancel:
                task.cancel()

            # Wait for cancellation with timeout
            try:
                await asyncio.wait_for(
                    asyncio.gather(*tasks_to_cancel, return_exceptions=True),
                    timeout=5.0
                )
            except asyncio.TimeoutError:
                self.app.print("Warning: Some tasks didn't cancel within timeout")

        # Close WebSocket connection
        if self.ws:
            with contextlib.suppress(Exception):
                await self.ws.close()
            self.ws = None

        # Cancel pending registrations
        for future in self.pending_registrations.values():
            if not future.done():
                future.cancel()
        self.pending_registrations.clear()

        # Clear state
        self.message_handler_tasks.clear()
        self.running_executions.clear()
        self.persistent_callbacks.clear()

        self.connection_task = None
        self.ping_task = None

        self.app.print("✅ Registry client shutdown completed")

cancel_execution(request_id) async

Cancel a running execution.

Source code in toolboxv2/mods/registry/client.py

async def cancel_execution(self, request_id: str) -> bool:
    """Cancel a running execution."""
    try:
        if request_id not in self.running_executions:
            self.app.print(f"❌ Execution {request_id} not found")
            return False

        execution_task = self.running_executions[request_id]

        if execution_task.done():
            self.app.print(f"⚠️  Execution {request_id} already completed")
            return True

        # Cancel the task
        execution_task.cancel()

        try:
            # Wait a moment for graceful cancellation
            await asyncio.wait_for(execution_task, timeout=5.0)
        except asyncio.CancelledError:
            self.app.print(f"✅ Execution {request_id} cancelled successfully")
        except asyncio.TimeoutError:
            self.app.print(f"⚠️  Execution {request_id} cancellation timeout - may still be running")
        except Exception as e:
            self.app.print(f"⚠️  Execution {request_id} cancellation resulted in exception: {e}")

        # Send cancellation notice to server
        try:
            if self.is_connected and self.ws and self.ws.open:
                cancellation_event = ProgressEvent(
                    event_type="execution_cancelled",
                    node_name="RegistryClient",
                    success=False,
                    metadata={
                        "request_id": request_id,
                        "cancellation_reason": "client_requested",
                        "timestamp": asyncio.get_event_loop().time()
                    }
                )

                cancellation_message = ExecutionResult(
                    request_id=request_id,
                    payload=cancellation_event.to_dict(),
                    is_final=True
                )

                await self._send_message('execution_result', cancellation_message.model_dump())

        except Exception as e:
            self.app.print(f"Failed to send cancellation notice to server: {e}")

        # Cleanup
        self.running_executions.pop(request_id, None)

        return True

    except Exception as e:
        self.app.print(f"Error cancelling execution {request_id}: {e}")
        return False

cleanup_completed_executions() async

Clean up completed execution tasks.

Source code in toolboxv2/mods/registry/client.py

async def cleanup_completed_executions(self):
    """Clean up completed execution tasks."""
    try:
        completed_tasks = []

        for request_id, task in self.running_executions.items():
            if task.done():
                completed_tasks.append(request_id)

        for request_id in completed_tasks:
            self.running_executions.pop(request_id, None)
            self.app.print(f"🧹 Cleaned up completed execution: {request_id}")

        return len(completed_tasks)

    except Exception as e:
        self.app.print(f"Error during cleanup: {e}")
        return 0

connect(server_url, timeout=30.0) async

Connect and start all background tasks.

Source code in toolboxv2/mods/registry/client.py

async def connect(self, server_url: str, timeout: float = 30.0):
    """Connect and start all background tasks."""
    if not ws_client:
        self.app.print("Websockets library not installed. Please run 'pip install websockets'")
        return False

    if self.ws and self.ws.open:
        self.app.print("Already connected to the registry server.")
        return True

    self.server_url = server_url
    self.should_reconnect = True
    self.reconnect_in_progress = False

    try:
        self.app.print(f"Connecting to Registry Server at {server_url}...")
        self.ws = await asyncio.wait_for(
            ws_client.connect(server_url),
            timeout=timeout
        )

        self.is_connected = True
        self.reconnect_attempts = 0

        # Start all background tasks
        await self._start_all_background_tasks()

        self.app.print(f"✅ Successfully connected and started all tasks")
        return True

    except asyncio.TimeoutError:
        self.app.print(f"❌ Connection timeout after {timeout}s")
        return False
    except Exception as e:
        self.app.print(f"❌ Connection failed: {e}")
        return False

disconnect() async

Enhanced disconnect with complete task cleanup.

Source code in toolboxv2/mods/registry/client.py

async def disconnect(self):
    """Enhanced disconnect with complete task cleanup."""
    self.app.print("Initiating clean shutdown...")
    self.is_connected = False
    self.should_reconnect = False

    # Cancel all background tasks
    tasks_to_cancel = []

    if self.connection_task and not self.connection_task.done():
        tasks_to_cancel.append(self.connection_task)

    if self.ping_task and not self.ping_task.done():
        tasks_to_cancel.append(self.ping_task)

    # Cancel message handler tasks
    for task in list(self.message_handler_tasks):
        if not task.done():
            tasks_to_cancel.append(task)

    # Cancel running executions
    for task in list(self.running_executions.values()):
        if not task.done():
            tasks_to_cancel.append(task)

    if tasks_to_cancel:
        self.app.print(f"Cancelling {len(tasks_to_cancel)} background tasks...")
        for task in tasks_to_cancel:
            task.cancel()

        # Wait for cancellation with timeout
        try:
            await asyncio.wait_for(
                asyncio.gather(*tasks_to_cancel, return_exceptions=True),
                timeout=5.0
            )
        except asyncio.TimeoutError:
            self.app.print("Warning: Some tasks didn't cancel within timeout")

    # Close WebSocket connection
    if self.ws:
        with contextlib.suppress(Exception):
            await self.ws.close()
        self.ws = None

    # Cancel pending registrations
    for future in self.pending_registrations.values():
        if not future.done():
            future.cancel()
    self.pending_registrations.clear()

    # Clear state
    self.message_handler_tasks.clear()
    self.running_executions.clear()
    self.persistent_callbacks.clear()

    self.connection_task = None
    self.ping_task = None

    self.app.print("✅ Registry client shutdown completed")

get_connection_status() async

Get detailed connection status information.

Source code in toolboxv2/mods/registry/client.py

async def get_connection_status(self) -> dict[str, Any]:
    """Get detailed connection status information."""
    try:
        connection_status = {
            "is_connected": self.is_connected,
            "server_url": self.server_url,
            "reconnect_attempts": self.reconnect_attempts,
            "max_reconnect_attempts": self.max_reconnect_attempts,
            "should_reconnect": self.should_reconnect,
            "reconnect_in_progress": self.reconnect_in_progress,
            "websocket_state": None,
            "websocket_open": False,
            "tasks": {
                "connection_task_running": self.connection_task and not self.connection_task.done(),
                "ping_task_running": self.ping_task and not self.ping_task.done(),
            },
            "registered_agents_count": len(self.local_agents),
            "running_executions_count": len(self.running_executions),
            "pending_registrations_count": len(self.pending_registrations),
            "persistent_callbacks_count": len(self.persistent_callbacks),
            "last_ping_time": getattr(self, 'last_ping_time', None),
            "connection_uptime": None,
            "connection_established_at": getattr(self, 'connection_established_at', None),
        }

        # WebSocket specific status
        if self.ws:
            connection_status.update({
                "websocket_state": str(self.ws.state.name) if hasattr(self.ws.state, 'name') else str(
                    self.ws.state),
                "websocket_open": self.ws.open,
                "websocket_closed": self.ws.closed,
            })

        # Calculate uptime
        if hasattr(self, 'connection_established_at') and self.connection_established_at:
            connection_status[
                "connection_uptime"] = asyncio.get_event_loop().time() - self.connection_established_at

        return connection_status

    except Exception as e:
        self.app.print(f"Error getting connection status: {e}")
        return {
            "error": str(e),
            "is_connected": False,
            "server_url": self.server_url,
        }

get_diagnostics() async

Get comprehensive diagnostic information.

Source code in toolboxv2/mods/registry/client.py

async def get_diagnostics(self) -> dict[str, Any]:
    """Get comprehensive diagnostic information."""
    try:
        diagnostics = {
            "connection_status": await self.get_connection_status(),
            "registered_agents": await self.get_registered_agents(),
            "running_executions": await self.get_running_executions(),
            "health_status": await self.health_check(),
            "system_info": {
                "python_version": sys.version,
                "asyncio_running": True,
                "event_loop": str(asyncio.get_running_loop()),
                "thread_name": threading.current_thread().name,
            },
            "performance_metrics": {
                "total_messages_sent": getattr(self, 'total_messages_sent', 0),
                "total_messages_received": getattr(self, 'total_messages_received', 0),
                "total_reconnections": self.reconnect_attempts,
                "total_registrations": len(self.registered_info),
                "memory_usage": self._get_memory_usage(),
            },
            "error_log": getattr(self, 'recent_errors', []),
        }

        return diagnostics

    except Exception as e:
        return {
            "diagnostics_error": str(e),
            "timestamp": asyncio.get_event_loop().time()
        }

get_registered_agents() async

Get all registered agents information.

Source code in toolboxv2/mods/registry/client.py

async def get_registered_agents(self) -> dict[str, AgentRegistered]:
    """Get all registered agents information."""
    try:
        agents_info = {}

        for agent_id, reg_info in self.registered_info.items():
            # Get agent instance if available
            agent_instance = self.local_agents.get(agent_id)

            # Create enhanced agent info
            agent_data = {
                "registration_info": reg_info,
                "agent_available": agent_instance is not None,
                "agent_type": type(agent_instance).__name__ if agent_instance else "Unknown",
                "has_progress_callback": hasattr(agent_instance, 'progress_callback') if agent_instance else False,
                "supports_progress_callback": hasattr(agent_instance,
                                                      'set_progress_callback') if agent_instance else False,
                "is_persistent_callback_active": agent_id in self.persistent_callbacks,
                "registration_timestamp": getattr(reg_info, 'registration_timestamp', None),
            }

            # Add agent capabilities if available
            if agent_instance and hasattr(agent_instance, 'get_capabilities'):
                try:
                    agent_data["capabilities"] = await agent_instance.get_capabilities()
                except Exception as e:
                    agent_data["capabilities_error"] = str(e)

            agents_info[agent_id] = agent_data

        return agents_info

    except Exception as e:
        self.app.print(f"Error getting registered agents: {e}")
        return {}

get_running_executions() async

Get information about currently running executions.

Source code in toolboxv2/mods/registry/client.py

async def get_running_executions(self) -> dict[str, dict[str, Any]]:
    """Get information about currently running executions."""
    try:
        executions_info = {}

        for request_id, execution_task in self.running_executions.items():
            execution_info = {
                "request_id": request_id,
                "task_done": execution_task.done(),
                "task_cancelled": execution_task.cancelled(),
                "start_time": getattr(execution_task, 'start_time', None),
                "running_time": None,
                "task_exception": None,
                "task_result": None,
            }

            # Calculate running time
            if hasattr(execution_task, 'start_time') and execution_task.start_time:
                execution_info["running_time"] = asyncio.get_event_loop().time() - execution_task.start_time

            # Get task status details
            if execution_task.done():
                try:
                    if execution_task.exception():
                        execution_info["task_exception"] = str(execution_task.exception())
                    else:
                        execution_info["task_result"] = "completed_successfully"
                except Exception as e:
                    execution_info["task_status_error"] = str(e)

            executions_info[request_id] = execution_info

        return executions_info

    except Exception as e:
        self.app.print(f"Error getting running executions: {e}")
        return {}

health_check() async

Perform a health check of the connection.

Source code in toolboxv2/mods/registry/client.py

async def health_check(self) -> bool:
    """Perform a health check of the connection."""
    try:
        # Basic connection checks
        if not self.is_connected:
            self.app.print("🔍 Health check: Not connected")
            return False

        if not self.ws or not self.ws.open:
            self.app.print("🔍 Health check: WebSocket not open")
            return False

        # Ping test
        try:
            pong_waiter = await self.ws.ping()
            await asyncio.wait_for(pong_waiter, timeout=10.0)

            # Update last ping time
            self.last_ping_time = asyncio.get_event_loop().time()

            # Test message sending
            test_message = WsMessage(
                event='health_check',
                data={
                    "timestamp": self.last_ping_time,
                    "client_id": getattr(self, 'client_id', 'unknown'),
                    "registered_agents": list(self.local_agents.keys()),
                    "running_executions": list(self.running_executions.keys())
                }
            )

            await self.ws.send(test_message.model_dump_json())

            self.app.print("✅ Health check: Connection healthy")
            return True

        except asyncio.TimeoutError:
            self.app.print("❌ Health check: Ping timeout")
            return False
        except Exception as ping_error:
            self.app.print(f"❌ Health check: Ping failed - {ping_error}")
            return False

    except Exception as e:
        self.app.print(f"❌ Health check: Error - {e}")
        return False

on(event_name, handler)

Register an async callback function to handle a custom event from the server.

Source code in toolboxv2/mods/registry/client.py

def on(self, event_name: str, handler: Callable[[dict], Awaitable[None]]):
    """Register an async callback function to handle a custom event from the server."""
    self.app.print(f"Handler for custom event '{event_name}' registered.")
    self.custom_event_handlers[event_name] = handler

register(agent_instance, public_name, description=None) async

Register an agent with the server.

Source code in toolboxv2/mods/registry/client.py

async def register(self, agent_instance: Any, public_name: str, description: str | None = None) -> AgentRegistered | None:
    """Register an agent with the server."""
    if not self.is_connected or not self.ws:
        self.app.print("Not connected. Cannot register agent.")
        return None

    try:
        # Create registration request
        registration = AgentRegistration(public_name=public_name, description=description)
        message = WsMessage(event='register', data=registration.model_dump())

        # Create future for registration response
        reg_id = f"reg_{self.registration_counter}"
        self.registration_counter += 1
        self.pending_registrations[reg_id] = asyncio.Future()

        # Send registration request
        await self.ws.send(message.model_dump_json())
        self.app.print(f"Sent registration request for agent '{public_name}'")

        # Wait for registration confirmation
        try:
            reg_info = await asyncio.wait_for(self.pending_registrations[reg_id], timeout=30.0)

            # Store agent and registration info
            self.local_agents[reg_info.public_agent_id] = agent_instance
            self.registered_info[reg_info.public_agent_id] = reg_info

            self.app.print(f"Agent '{public_name}' registered successfully.")
            self.app.print(f"  Public URL: {reg_info.public_url}")
            self.app.print(f"  API Key: {reg_info.public_api_key}")

            return reg_info

        except TimeoutError:
            self.app.print("Timeout waiting for registration confirmation.")
            return None

    except Exception as e:
        self.app.print(f"Error during registration: {e}")
        return None
    finally:
        # Cleanup pending registration
        self.pending_registrations.pop(reg_id, None)

send_agent_status(agent_id, status, details=None) async

Send agent status updates.

Source code in toolboxv2/mods/registry/client.py

async def send_agent_status(self, agent_id: str, status: str, details: dict[str, Any] = None):
    """Send agent status updates."""
    if not self.is_connected or not self.ws or not self.ws.open:
        return

    try:
        status_message = {
            "agent_id": agent_id,
            "status": status,
            "details": details or {},
            "timestamp": asyncio.get_event_loop().time(),
            "capabilities": ["chat", "progress_tracking", "outline_visualization", "meta_tool_monitoring"]
        }

        message = WsMessage(event='agent_status_update', data=status_message)
        await self.ws.send(message.model_dump_json())

    except Exception as e:
        self.app.print(f"Failed to send agent status: {e}")
        await self._handle_connection_error()

send_custom_event(event_name, data) async

Send a custom event with a JSON payload to the server.

Source code in toolboxv2/mods/registry/client.py

async def send_custom_event(self, event_name: str, data: dict[str, Any]):
    """Send a custom event with a JSON payload to the server."""
    if not self.is_connected or not self.ws or not self.ws.open:
        self.app.print("Cannot send custom event: Not connected.")
        return

    try:
        message = WsMessage(event=event_name, data=data)
        await self.ws.send(message.model_dump_json())
        self.app.print(f"Sent custom event '{event_name}' to server.")
    except Exception as e:
        self.app.print(f"Failed to send custom event: {e}")
        await self._handle_connection_error()

send_ui_progress(progress_data, retry_count=3) async

Enhanced UI progress sender with retry logic.

Source code in toolboxv2/mods/registry/client.py

async def send_ui_progress(self, progress_data: dict[str, Any], retry_count: int = 3):
    """Enhanced UI progress sender with retry logic."""
    if not self.is_connected or not self.ws or not self.ws.open:
        self.app.print("Registry client WebSocket not connected - queuing progress update")
        # Could implement a queue here for offline progress updates
        return False

    for attempt in range(retry_count):
        try:
            # Structure progress message for registry server
            ui_message = {
                "timestamp": progress_data.get('timestamp', asyncio.get_event_loop().time()),
                "agent_id": progress_data.get('agent_id', 'unknown'),
                "event_type": progress_data.get('event_type', 'unknown'),
                "status": progress_data.get('status', 'processing'),
                "agent_name": progress_data.get('agent_name', 'Unknown'),
                "node_name": progress_data.get('node_name', 'Unknown'),
                "session_id": progress_data.get('session_id'),
                "metadata": progress_data.get('metadata', {}),

                # Enhanced progress data for UI panels
                "outline_progress": progress_data.get('progress_data', {}).get('outline', {}),
                "activity_info": progress_data.get('progress_data', {}).get('activity', {}),
                "meta_tool_info": progress_data.get('progress_data', {}).get('meta_tool', {}),
                "system_status": progress_data.get('progress_data', {}).get('system', {}),
                "graph_info": progress_data.get('progress_data', {}).get('graph', {}),

                # UI flags for selective updates
                "ui_flags": progress_data.get('ui_flags', {}),

                # Performance metrics
                "performance": progress_data.get('performance', {}),

                # Message metadata
                "message_id": f"msg_{asyncio.get_event_loop().time()}_{attempt}",
                "retry_count": attempt
            }

            # Send as WsMessage
            message = WsMessage(event='ui_progress_update', data=ui_message)
            await self.ws.send(message.model_dump_json())

            # Success - break retry loop
            self.app.print(
                f"📤 Sent UI progress: {progress_data.get('event_type')} | {progress_data.get('status')} (attempt {attempt + 1})")
            return True

        except Exception as e:
            self.app.print(f"Failed to send UI progress (attempt {attempt + 1}/{retry_count}): {e}")
            if attempt < retry_count - 1:
                await asyncio.sleep(0.5 * (attempt + 1))  # Exponential backoff
            else:
                await self._handle_connection_error()
                return False

    return False

get_registry_client(app)

Factory function to get a singleton RegistryClient instance.

Source code in toolboxv2/mods/registry/client.py

def get_registry_client(app: App) -> RegistryClient:
    """Factory function to get a singleton RegistryClient instance."""
    app_id = app.id
    if app_id not in registry_clients:
        registry_clients[app_id] = RegistryClient(app)
    return registry_clients[app_id]

demo_custom_messaging

setup_chain_with_live_updates() async

Example 3: Create agent chain with live progress broadcasting

Source code in toolboxv2/mods/registry/demo_custom_messaging.py

async def setup_chain_with_live_updates():
    """Example 3: Create agent chain with live progress broadcasting"""
    app = get_app("ChainLiveExample")
    isaa = app.get_mod("isaa")

    # Initialize ISAA
    await isaa.init_isaa()

    # Create and register specialized agents

    # Research agent
    researcher_builder = isaa.get_agent_builder("researcher_agent")
    researcher_builder.with_system_message(
        "You are a research specialist. Gather comprehensive information and provide detailed analysis. "
        "Always report your progress clearly."
    )
    #researcher_builder.with_models(complex_llm_model="openrouter/openai/gpt-4o")
    await isaa.register_agent(researcher_builder)

    # Writer agent
    writer_builder = isaa.get_agent_builder("writer_agent")
    writer_builder.with_system_message(
        "You are a professional writer. Create well-structured, engaging content from research data. "
        "Report your writing progress step by step."
    )
    #writer_builder.with_models(complex_llm_model="openrouter/openai/gpt-4o")
    await isaa.register_agent(writer_builder)

    # Reviewer agent
    reviewer_builder = isaa.get_agent_builder("reviewer_agent")
    reviewer_builder.with_system_message(
        "You are a quality reviewer. Check for accuracy, completeness, and suggest improvements. "
        "Report your review progress clearly."
    )
    # reviewer_builder.with_models(fast_llm_model="openrouter/anthropic/claude-3-haiku")
    await isaa.register_agent(reviewer_builder)

    # Get agent instances
    researcher = await isaa.get_agent("researcher_agent")
    writer = await isaa.get_agent("writer_agent")
    reviewer = await isaa.get_agent("reviewer_agent")

    # Create chain using the >> operator for sequential execution
    from pydantic import BaseModel
    class Topick(BaseModel):
        topic: str

    class MiniBlog(BaseModel):
        title: str
        content: str

    class Review(BaseModel):
        feedback: str
        better_title: str
        better_content: str

    chain = researcher >> CF(Topick) >> writer >> CF(MiniBlog) >> reviewer >> CF(Review)
    chain.name = "content_creation_chain"

    # Publish chain with live updates - Progress Callback wird automatisch eingerichtet
    result = await isaa.publish_and_host_agent(
        agent=chain,
        public_name="Content Creation Pipeline",
        description="Multi-agent chain with live progress: Research → Write → Review",
        registry_server="ws://localhost:8080/ws/registry/connect",
    )

    if result.get('public_url'):
        app.print("🔗 Chain published successfully with Live Progress UI!")
        app.print(f"   Local UI: {result['ui_url']}")
        app.print(f"   WebSocket: {result.get('registry_server')}")
        app.print(f"   WebSocket: {result.get('websocket_url')}")
        app.print(f"   Public URL: {result.get('public_url')}")
        app.print(f"   API Key: {result.get('public_api_key')}")
        print(result)

        # Example usage - test the chain with live updates
        #pp.print("\n🧪 Testing chain execution with live progress tracking:")
        #ry:
        #   result_text = await chain.a_run(
        #       query="Create a comprehensive article about renewable energy trends in 2024",
        #       session_id="demo-session"
        #   )
        #   app.print(f"✅ Chain completed successfully!")
        #   app.print(f"   Result length: {len(result_text)} characters")
        #   app.print("   All progress was tracked live in the UI!")
        #xcept Exception as e:
        #   app.print(f"❌ Chain execution failed: {e}")

        # Keep services running with live status
        try:
            while True:
                await asyncio.sleep(30)
                app.print("💓 Chain services live - ready for requests")
        except KeyboardInterrupt:
            app.print("Shutting down chain services...")
    else:
        app.print("❌ Failed to publish chain to registry")

    # Clean shutdown
    await researcher.close()
    await writer.close()
    await reviewer.close()

setup_complete_agent_system(local=False) async

Vollständiges Beispiel für Agent-System mit Live-Progress.

Source code in toolboxv2/mods/registry/demo_custom_messaging.py

async def setup_complete_agent_system(local=False):
    """Vollständiges Beispiel für Agent-System mit Live-Progress."""

    app = get_app("CompleteAgentSystem")
    isaa = app.get_mod("isaa")

    # ISAA initialisieren
    await isaa.init_isaa()

    # Erweiterten Agent erstellen
    advanced_builder = isaa.get_agent_builder("production_assistant")
    advanced_builder.with_system_message("""
        Du bist ein produktions-fertiger AI-Assistent mit detailliertem Progress-Tracking.

        Arbeitsweise:
        1. Analysiere die Anfrage sorgfältig
        2. Erstelle einen strukturierten Plan (Outline)
        3. Führe jeden Schritt methodisch aus
        4. Verwende Meta-Tools für komplexe Aufgaben
        5. Berichte kontinuierlich über deinen Fortschritt
        6. Liefere umfassende, gut strukturierte Antworten

        Zeige immer, welche Tools du verwendest und warum.
        Erkläre deine Reasoning-Loops transparent.
        """)

    # Agent registrieren
    await isaa.register_agent(advanced_builder)
    agent = await isaa.get_agent("production_assistant")

    # **Produktionsfertige Publish & Host - Ein Aufruf macht alles**
    result = await isaa.publish_and_host_agent(
        agent=agent,
        public_name="Production AI Assistant",
        registry_server="ws://localhost:8080/ws/registry/connect" if local else "wss://simplecore.app/ws/registry/connect",
        description="Production-ready AI assistant with comprehensive progress tracking, step-by-step reasoning, and meta-tool visualization. Supports real-time progress updates, outline tracking, and multi-user access.",
        access_level="public"
    )

    if result.get('success'):
        app.print("🎉 AGENT SYSTEM FULLY DEPLOYED!")
        app.print("")
        app.print("🌐 Public Access:")
        app.print(f"   URL: {result['public_url']}")
        app.print(f"   API Key: {result['public_api_key']}")
        app.print("")
        app.print("🖥️  Live UI:")
        app.print(f"   Registry UI: {result['ui_url']}")
        if result.get('local_ui'):
            app.print(f"   Local UI: {result['local_ui'].get('ui_url')}")
        app.print("")
        app.print("🔌 WebSocket:")
        app.print(f"   Live Updates: {result['websocket_url']}")
        app.print("")
        app.print("📋 cURL Test:")
        app.print(f"""curl -X POST {result['public_url']} \\
  -H "Content-Type: application/json" \\
  -H "Authorization: Bearer {result['public_api_key']}" \\
  -d '{{"query": "Create a detailed analysis of quantum computing with step-by-step progress", "session_id": "test-session"}}'""")

        # Lokaler Test des Agents
        app.print("\n🧪 Testing agent locally...")
        #await asyncio.sleep(5)
        #test_result = await agent.a_run(
        #    "hey",
        #    session_id="local_test"
        #)
        app.print("✅ Test completed successfully!")

        # Service am Leben halten
        try:
            while True:
                await asyncio.sleep(30)
                app.print("💓 Agent services running - ready for requests")
        except KeyboardInterrupt:
            app.print("🛑 Shutting down agent services...")
    else:
        app.print(f"❌ Deployment failed: {result.get('error')}")
        print(result)

    await agent.close()

setup_multiple_live_agents() async

Example 4: Host multiple agents with individual live UIs

Source code in toolboxv2/mods/registry/demo_custom_messaging.py

async def setup_multiple_live_agents():
    """Example 4: Host multiple agents with individual live UIs"""
    app = get_app("MultiAgentLiveExample")
    isaa = app.get_mod("isaa")

    # Initialize ISAA
    await isaa.init_isaa()

    # Create different specialized agents
    agents_config = [
        {
            "name": "math_tutor",
            "system": "You are a mathematics tutor. Explain concepts step-by-step with live progress updates.",
            "public_name": "Live Math Tutor",
            "port": 8770
        },
        {
            "name": "code_helper",
            "system": "You are a coding assistant. Help debug and explain code with detailed progress tracking.",
            "public_name": "Live Code Assistant",
            "port": 8771
        },
        {
            "name": "creative_writer",
            "system": "You are a creative writer. Generate stories and content with live creative process updates.",
            "public_name": "Live Creative Writer",
            "port": 8772
        }
    ]

    hosted_agents = []

    # Create and host each agent
    for config in agents_config:
        # Create agent builder
        builder = isaa.get_agent_builder(config["name"])
        builder.with_system_message(config["system"])
        # builder.with_models(complex_llm_model="openrouter/openai/gpt-4o")

        # Register agent
        await isaa.register_agent(builder)

        # Get agent instance
        agent = await isaa.get_agent(config["name"])

        # Host with live UI - Progress wird automatisch eingerichtet
        result = await isaa.publish_and_host_agent(
            agent=agent,
            public_name=config["public_name"],
            description=f"Specialized agent: {config['public_name']} with live progress updates",
        )

        hosted_agents.append({
            'name': config["name"],
            'agent': agent,
            'result': result
        })

        app.print(f"🚀 {config['public_name']} live at: {result['ui_url']}")

    # Test all agents with live progress
    app.print("\n🧪 Testing all agents with live progress:")

    test_queries = [
        ("math_tutor", "Explain how to solve quadratic equations step by step"),
        ("code_helper", "Debug this Python function and explain the process"),
        ("creative_writer", "Write a short story about AI and humans working together")
    ]

    for agent_name, query in test_queries:
        agent_info = next(a for a in hosted_agents if a['name'] == agent_name)
        app.print(f"Testing {agent_name} - watch live progress in UI...")

        try:
            result = await agent_info['agent'].a_run(query, session_id=f"test_{agent_name}")
            app.print(f"✅ {agent_name} completed - live progress was shown!")
        except Exception as e:
            app.print(f"❌ {agent_name} failed: {e}")

    # Keep all agents running
    try:
        while True:
            await asyncio.sleep(60)
            app.print("💓 All agents live and ready")
            for agent_info in hosted_agents:
                app.print(f"   • {agent_info['name']}: {agent_info['result']['ui_url']}")
    except KeyboardInterrupt:
        app.print("Shutting down all live agents...")
        for agent_info in hosted_agents:
            await agent_info['agent'].close()

demo_registry

run_end_user_test() async

Simuliert einen externen Aufruf an die öffentliche API des Registry Servers.

Source code in toolboxv2/mods/registry/demo_registry.py

async def run_end_user_test():
    """Simuliert einen externen Aufruf an die öffentliche API des Registry Servers."""
    print("--- [USER] Warte darauf, dass der Agent publiziert wird... ---")
    await published_event.wait()
    print("--- [USER] Agent ist jetzt öffentlich. Starte Testaufruf in 3 Sekunden... ---")
    await asyncio.sleep(3)

    public_url = published_info.get("public_url")
    api_key = published_info.get("public_api_key")

    if not public_url or not api_key:
        print("--- [USER] FEHLER: Keine öffentlichen Agenten-Infos gefunden!", file=sys.stderr)
        return

    print(f"--- [USER] Sende POST-Anfrage an: {public_url} ---")

    request_payload = {
        "query": "Hallo, weitergeleitete Welt!",
        "session_id": "ext-user-session-001"
    }

    headers = {
        "Authorization": f"Bearer {api_key}",
        "Content-Type": "application/json"
    }

    async with aiohttp.ClientSession() as session:
        try:
            async with session.post(public_url, json=request_payload, headers=headers) as response:
                print(f"--- [USER] Antwort-Status: {response.status} ---")

                if response.status == 200:
                    print("--- [USER] Beginne mit dem Streamen der Antwort-Events: ---")
                    # Die Antwort ist application/json-seq, also lesen wir zeilenweise
                    async for line in response.content:
                        if line:
                            try:
                                data = json.loads(line)
                                event_type = data.get('event_type', 'unknown')
                                status = data.get('status', '...')
                                print(f"  [STREAM] Event: {event_type:<20} | Status: {status} {data}")

                                # Der finale Event enthält das Ergebnis
                                if event_type == "final_result":
                                    final_result = data.get('details', {}).get('result')
                                    print("\n--- [USER] Endgültiges Ergebnis erhalten: ---")
                                    print(f"  >>> {final_result}")

                            except json.JSONDecodeError:
                                print(f"  [STREAM] Konnte Zeile nicht als JSON parsen: {line.decode()}")
                else:
                    error_text = await response.text()
                    print(f"--- [USER] FEHLER vom Server: {error_text}", file=sys.stderr)
        except aiohttp.ClientConnectorError as e:
            print(f"--- [USER] VERBINDUNGSFEHLER: Konnte den Server nicht erreichen. Läuft er? Fehler: {e}",
                  file=sys.stderr)

run_local_client() async

Startet die zweite toolboxv2-Instanz als lokalen Client, der einen Agenten hostet.

Source code in toolboxv2/mods/registry/demo_registry.py

async def run_local_client():
    """Startet die zweite toolboxv2-Instanz als lokalen Client, der einen Agenten hostet."""
    print("--- [CLIENT] Initialisiere lokale Client Instanz ---")
    client_app = get_app("LocalClientInstance")

    # ISAA-Modul für diese Instanz holen und initialisieren
    isaa: ISAA_Tools = client_app.get_mod("isaa")
    await isaa.init_isaa()
    print("--- [CLIENT] ISAA initialisiert. ---")

    # --- Agenten erstellen ---
    print("--- [CLIENT] Erstelle einen einfachen 'EchoAgent'... ---")
    builder = isaa.get_agent_builder("EchoAgent")
    builder.with_system_message("You are an echo agent. Repeat the user's query exactly, but prefix it with 'Echo: '.")
    await isaa.register_agent(builder)

    # Agenten-Instanz holen (dieser Schritt ist nicht zwingend für das Publizieren per Name, aber gut zur Demo)
    echo_agent = await isaa.get_agent("EchoAgent")
    print(f"--- [CLIENT] 'EchoAgent' ({type(echo_agent).__name__}) erstellt. ---")

    # --- Agenten publizieren ---
    # Warten, bis der Server sicher läuft
    await asyncio.sleep(2)

    server_ws_url = "ws://127.0.0.1:8080/ws/registry/connect"
    print(f"--- [CLIENT] Publiziert 'EchoAgent' am Server: {server_ws_url} ---")

    # Die neue `publish_agent` Methode aufrufen
    reg_info = await isaa.host_agent_ui(
        agent=echo_agent,
        public_name="Public Echo Service",
        server_url=server_ws_url,
        description="A simple agent that echoes your input."
    )

    if reg_info:
        print("--- [CLIENT] Agent erfolgreich publiziert! Details erhalten: ---")
        print(f"  > Public URL: {reg_info.public_url}")
        print(f"  > API Key: {reg_info.public_api_key}")

        # Speichere die Info und signalisiere dem Endbenutzer-Task, dass er starten kann
        published_info.update(reg_info.model_dump())
        published_event.set()
    else:
        print("--- [CLIENT] FEHLER: Agenten-Publizierung fehlgeschlagen. ---", file=sys.stderr)

    # Hält diesen Task am Leben, um auf Weiterleitungsanfragen zu lauschen.
    await asyncio.Future()

run_registry_server() async

Startet die erste toolboxv2-Instanz als unseren öffentlichen Server.

Source code in toolboxv2/mods/registry/demo_registry.py

async def run_registry_server():
    """Startet die erste toolboxv2-Instanz als unseren öffentlichen Server."""
    print("--- [SERVER] Initialisiere Registry Server Instanz ---")

    # Holt sich eine App-Instanz. Das Laden des 'registry'-Moduls geschieht
    # automatisch durch die __init__.py-Struktur von toolboxv2.
    server_app = get_app("RegistryServerInstance")

    # Startet den actix-web Server auf Port 8080.
    # `blocking=False` ist entscheidend, damit asyncio weiterlaufen kann.
    server_app.start_server()

    print("--- [SERVER] Registry Server läuft auf http://127.0.0.1:8080 ---")
    print("--- [SERVER] Wartet auf eingehende Client-Verbindungen... ---")

    # Hält diesen Task am Leben, um den Server laufen zu lassen.
    await asyncio.Future()

server

broadcast_to_ui_clients(app, data) async

Broadcast updates to all connected UI clients.

Source code in toolboxv2/mods/registry/server.py

async def broadcast_to_ui_clients(app: App, data: dict[str, Any]):
    """Broadcast updates to all connected UI clients."""
    if not STATE.ui_clients:
        app.print("No active UI clients to broadcast to")
        return

    app.print(f"Broadcasting to {len(STATE.ui_clients)} UI clients: {data.get('event', 'unknown')}")

    dead_clients = set()
    successful_broadcasts = 0

    for ui_conn_id in STATE.ui_clients.copy():
        try:
            await app.ws_send(ui_conn_id, data)
            successful_broadcasts += 1
        except Exception as e:
            app.print(f"Failed to broadcast to UI client {ui_conn_id}: {e}")
            dead_clients.add(ui_conn_id)

    # Clean up dead connections
    for dead_client in dead_clients:
        STATE.ui_clients.discard(dead_client)

    app.print(f"Broadcast completed: {successful_broadcasts} successful, {len(dead_clients)} failed")

handle_agent_status_update(app, message) async

Handle agent status updates.

Source code in toolboxv2/mods/registry/server.py

async def handle_agent_status_update(app: App, message: WsMessage):
    """Handle agent status updates."""
    try:
        status_data = message.data
        await broadcast_to_ui_clients(app, {
            'event': 'agent_status_update',
            'data': status_data
        })

    except Exception as e:
        app.print(f"Agent status update error: {e}", error=True)

handle_execution_error(app, message) async

Handle execution errors.

Source code in toolboxv2/mods/registry/server.py

async def handle_execution_error(app: App, message: WsMessage):
    """Handle execution errors."""
    try:
        error = ExecutionError.model_validate(message.data)

        if error.request_id in STATE.pending_requests:
            await STATE.pending_requests[error.request_id].put(error)

        await broadcast_to_ui_clients(app, {
            'event': 'execution_error',
            'data': {
                'request_id': error.request_id,
                'error': error.error,
                'timestamp': asyncio.get_event_loop().time()
            }
        })

    except Exception as e:
        app.print(f"Execution error handling error: {e}", error=True)

handle_execution_result(app, message) async

Handle execution results.

Source code in toolboxv2/mods/registry/server.py

async def handle_execution_result(app: App, message: WsMessage):
    """Handle execution results."""
    try:
        result = ExecutionResult.model_validate(message.data)

        if result.request_id in STATE.pending_requests:
            await STATE.pending_requests[result.request_id].put(result)

        # Broadcast to UI clients
        await broadcast_to_ui_clients(app, {
            'event': 'execution_progress',
            'data': {
                'request_id': result.request_id,
                'payload': result.payload,
                'is_final': result.is_final,
                'timestamp': asyncio.get_event_loop().time()
            }
        })

    except Exception as e:
        app.print(f"Execution result error: {e}", error=True)

handle_registration(app, conn_id, session, message) async

Handle agent registration.

Source code in toolboxv2/mods/registry/server.py

async def handle_registration(app: App, conn_id: str, session: dict, message: WsMessage):
    """Handle agent registration."""
    try:
        reg_data = AgentRegistration.model_validate(message.data)
        agent_id = f"agent_{secrets.token_urlsafe(16)}"
        api_key = f"tbk_{secrets.token_urlsafe(32)}"

        STATE.client_agents.setdefault(conn_id, []).append(agent_id)
        STATE.agent_to_client[agent_id] = conn_id
        STATE.key_to_agent[api_key] = agent_id
        STATE.agent_details[agent_id] = reg_data.model_dump()

        base_url = os.getenv("APP_BASE_URL", "http://localhost:8080") or session.get('host', 'localhost:8080')
        if base_url == "localhost":
            base_url = "localhost:8080"
            app.print("APP_BASE_URL is localhost. Using default port 8080.")
        public_url = f"{base_url}/api/registry/run?public_agent_id={agent_id}"

        if not public_url.startswith('http'):
            public_url = f"http://{public_url}"

        response = AgentRegistered(
            public_name=reg_data.public_name,
            public_agent_id=agent_id,
            public_api_key=api_key,
            public_url=public_url,
        )

        # Send registration confirmation
        response_message = WsMessage(event='agent_registered', data=response.model_dump())
        await app.ws_send(conn_id, response_message.model_dump())

        # Notify UI clients
        await broadcast_to_ui_clients(app, {
            "event": "agent_registered",
            "data": {
                "public_agent_id": agent_id,
                "public_name": reg_data.public_name,
                "description": reg_data.description,
                "status": "online"
            }
        })

        app.print(f"Agent '{reg_data.public_name}' registered with ID: {agent_id}")

    except Exception as e:
        app.print(f"Registration error: {e}", error=True)

handle_ui_progress_update(app, message) async

Handle UI progress updates.

Source code in toolboxv2/mods/registry/server.py

async def handle_ui_progress_update(app: App, message: WsMessage):
    """Handle UI progress updates."""
    try:
        progress_data = message.data
        agent_id = progress_data.get('agent_id', 'unknown')

        # Store recent progress
        if agent_id not in STATE.recent_progress:
            STATE.recent_progress[agent_id] = []
        STATE.recent_progress[agent_id].append(progress_data)

        # Keep only last 50 events
        STATE.recent_progress[agent_id] = STATE.recent_progress[agent_id][-50:]

        # Broadcast to UI clients
        await broadcast_to_ui_clients(app, {
            "event": "live_progress_update",
            "data": progress_data
        })

    except Exception as e:
        app.print(f"UI progress update error: {e}", error=True)

on_disconnect(app, conn_id, session=None) async

Enhanced disconnect handler with comprehensive cleanup and UI notifications.

Source code in toolboxv2/mods/registry/server.py

async def on_disconnect(app: App, conn_id: str, session: dict = None):
    """Enhanced disconnect handler with comprehensive cleanup and UI notifications."""
    app.print(f"Registry client disconnected: {conn_id}")

    # Check if this is a UI client
    if conn_id in STATE.ui_clients:
        STATE.ui_clients.discard(conn_id)
        app.print(f"UI client {conn_id} removed from active clients")
        return

    # Handle agent client disconnection
    if conn_id in STATE.client_agents:
        agent_ids_to_cleanup = STATE.client_agents[conn_id].copy()

        for agent_id in agent_ids_to_cleanup:
            try:
                # Get agent details before removal for notification
                agent_details = STATE.agent_details.get(agent_id, {})
                agent_name = agent_details.get('public_name', 'Unknown')

                # Remove from all state dictionaries
                STATE.agent_to_client.pop(agent_id, None)
                STATE.agent_details.pop(agent_id, None)

                # Remove API key mapping
                key_to_remove = next((k for k, v in STATE.key_to_agent.items() if v == agent_id), None)
                if key_to_remove:
                    STATE.key_to_agent.pop(key_to_remove, None)

                # Clean up progress data
                STATE.recent_progress.pop(agent_id, None)

                # Clean up any pending requests for this agent by checking if queue exists and clearing it
                requests_to_cleanup = []
                for req_id in list(STATE.pending_requests.keys()):
                    try:
                        # Put error in queue to unblock any waiting requests
                        error_result = ExecutionError(
                            request_id=req_id,
                            error="Agent disconnected unexpectedly",
                            public_agent_id=agent_id
                        )
                        await STATE.pending_requests[req_id].put(error_result)
                        requests_to_cleanup.append(req_id)
                    except Exception as e:
                        app.print(f"Error cleaning up pending request {req_id}: {e}")

                # Remove cleaned up requests
                for req_id in requests_to_cleanup:
                    STATE.pending_requests.pop(req_id, None)

                # Notify UI clients about agent going offline (non-blocking)
                if agent_details:
                    asyncio.create_task(broadcast_to_ui_clients(app, {
                        "event": "agent_offline",
                        "data": {
                            "public_agent_id": agent_id,
                            "public_name": agent_name,
                            "status": "offline",
                            "timestamp": asyncio.get_event_loop().time()
                        }
                    }))

                app.print(f"Agent '{agent_name}' (ID: {agent_id}) unregistered and cleaned up")

            except Exception as e:
                app.print(f"Error during agent cleanup for {agent_id}: {e}", error=True)

        # Remove the client connection entry
        STATE.client_agents.pop(conn_id, None)

        app.print(f"Client {conn_id} fully disconnected and cleaned up ({len(agent_ids_to_cleanup)} agents removed)")
    else:
        app.print(f"Unknown client {conn_id} disconnected (no agents to clean up)")

on_message(app, conn_id, session, payload) async

Enhanced message handler with proper error handling.

Source code in toolboxv2/mods/registry/server.py

async def on_message(app: App, conn_id: str, session: dict, payload: dict):
    """Enhanced message handler with proper error handling."""
    try:
        # Ensure payload is a dict
        if isinstance(payload, str):
            payload = json.loads(payload)

        message = WsMessage.model_validate(payload)
        app.print(f"Registry received event: {message.event} from {conn_id}")

        if message.event == 'register':
            await handle_registration(app, conn_id, session, message)

        elif message.event == 'ui_progress_update':
            await handle_ui_progress_update(app, message)

        elif message.event == 'execution_result':
            await handle_execution_result(app, message)

        elif message.event == 'execution_error':
            await handle_execution_error(app, message)

        elif message.event == 'agent_status_update':
            await handle_agent_status_update(app, message)

        else:
            app.print(f"Unhandled event '{message.event}' from client {conn_id}")

    except Exception as e:
        app.print(f"Error processing WebSocket message: {e}", error=True)

register_ui_ws_handlers(app)

Register UI-specific WebSocket handlers.

Source code in toolboxv2/mods/registry/server.py

@export(mod_name=Name, websocket_handler="ui_connect")
def register_ui_ws_handlers(app: App):
    """Register UI-specific WebSocket handlers."""
    return {
        "on_connect": ui_on_connect,
        "on_message": ui_on_message,
        "on_disconnect": ui_on_disconnect,
    }

register_ws_handlers(app)

Register WebSocket handlers for the registry.

Source code in toolboxv2/mods/registry/server.py

@export(mod_name=Name, websocket_handler="connect")
def register_ws_handlers(app: App):
    """Register WebSocket handlers for the registry."""
    return {
        "on_connect": on_connect,
        "on_message": on_message,
        "on_disconnect": on_disconnect,
    }

run(app, public_agent_id, request) async

Public API endpoint to run agents.

Source code in toolboxv2/mods/registry/server.py

@export(mod_name=Name, api=True, version="1", request_as_kwarg=True, api_methods=['POST'])
async def run(app: App, public_agent_id: str, request: RequestData):
    """Public API endpoint to run agents."""
    if request is None:
        return Result.default_user_error(info="Failed to run agent: No request provided.")
    if not request.headers:
        return Result.default_user_error(info="Failed to run agent: No request headers provided.")

    auth_header = request.headers.authorization or request.headers.to_dict().get('authorization')

    if not auth_header or not auth_header.startswith('Bearer '):
        return Result.default_user_error("Authorization header missing or invalid.", exec_code=401)

    api_key = auth_header.split(' ')[1]

    if STATE.key_to_agent.get(api_key) != public_agent_id:
        return Result.default_user_error("Invalid API Key or Agent ID.", exec_code=403)

    conn_id = STATE.agent_to_client.get(public_agent_id)
    if not conn_id:
        return Result.default_internal_error("Agent is not currently connected/online.", exec_code=503)

    body = request.body
    request_id = f"req_{secrets.token_urlsafe(16)}"

    run_request = RunRequest(
        request_id=request_id,
        public_agent_id=public_agent_id,
        query=body.get('query', ''),
        session_id=body.get('session_id'),
        kwargs=body.get('kwargs', {})
    )

    response_queue = asyncio.Queue()
    STATE.pending_requests[request_id] = response_queue

    # Send run request to the client
    await app.ws_send(conn_id, WsMessage(event='run_request', data=run_request.model_dump()).model_dump())

    try:
        final_result = None
        while True:
            item = await asyncio.wait_for(response_queue.get(), timeout=120.0)

            if isinstance(item, ExecutionError):
                return Result.default_internal_error(
                    info=f"An error occurred during agent execution: {item.error}",
                    exec_code=500
                )

            if item.is_final:
                final_result = item.payload.get("details", {}).get("result")
                break

        return Result.json(data={"result": final_result})

    except TimeoutError:
        return Result.default_internal_error(
            info="The request timed out as the agent did not respond in time.",
            exec_code=504
        )
    finally:
        STATE.pending_requests.pop(request_id, None)

ui(app, public_agent_id=None) async

Serve the interactive 3-panel agent UI.

Source code in toolboxv2/mods/registry/server.py

@export(mod_name=Name, api=True, version="1", api_methods=['GET'])
async def ui(app: App, public_agent_id: str = None):
    """Serve the interactive 3-panel agent UI."""
    from ..isaa.ui import get_agent_ui_html
    html_content = get_agent_ui_html()
    return Result.html(data=html_content, row=True)

ui_on_connect(app, conn_id, session) async

UI Client connection.

Source code in toolboxv2/mods/registry/server.py

async def ui_on_connect(app: App, conn_id: str, session: dict):
    """UI Client connection."""
    app.print(f"UI Client connecting: {conn_id}")
    STATE.ui_clients.add(conn_id)
    app.print(f"UI Client connected: {conn_id} (Total: {len(STATE.ui_clients)})")

    # Send current agents list
    available_agents = []
    for agent_id, details in STATE.agent_details.items():
        if agent_id in STATE.agent_to_client:
            available_agents.append({
                "public_agent_id": agent_id,
                "public_name": details.get('public_name', 'Unknown'),
                "description": details.get('description', ''),
                "status": "online"
            })

    await app.ws_send(conn_id, {
        "event": "agents_list",
        "data": {"agents": available_agents}
    })

ui_on_disconnect(app, conn_id, session=None) async

UI Client Disconnection.

Source code in toolboxv2/mods/registry/server.py

async def ui_on_disconnect(app: App, conn_id: str, session: dict = None):
    """UI Client Disconnection."""
    app.print(f"UI Client disconnected: {conn_id}")
    STATE.ui_clients.discard(conn_id)

ui_on_message(app, conn_id, session, payload) async

UI Client Message Handler.

Source code in toolboxv2/mods/registry/server.py

async def ui_on_message(app: App, conn_id: str, session: dict, payload: dict):
    """UI Client Message Handler."""
    try:
        # Ensure payload is a dict
        if isinstance(payload, str):
            payload = json.loads(payload)

        event = payload.get('event')
        data = payload.get('data', {})

        if event == 'subscribe_agent':
            agent_id = data.get('public_agent_id')
            if agent_id in STATE.agent_details:
                if agent_id in STATE.recent_progress:
                    for progress_event in STATE.recent_progress[agent_id][-10:]:
                        await app.ws_send(conn_id, {
                            "event": "historical_progress",
                            "data": progress_event
                        })

                await app.ws_send(conn_id, {
                    "event": "subscription_confirmed",
                    "data": {"public_agent_id": agent_id}
                })

        elif event == 'chat_message':
            agent_id = data.get('public_agent_id')
            message_text = data.get('message')
            session_id = data.get('session_id', f'ui_{conn_id}')
            api_key = data.get('api_key')

            if not api_key or STATE.key_to_agent.get(api_key) != agent_id:
                await app.ws_send(conn_id, {
                    "event": "error",
                    "data": {"error": "Invalid or missing API Key"}
                })
                return

            if agent_id in STATE.agent_to_client:
                agent_conn_id = STATE.agent_to_client[agent_id]
                request_id = f"ui_req_{secrets.token_urlsafe(16)}"

                run_request = RunRequest(
                    request_id=request_id,
                    public_agent_id=agent_id,
                    query=message_text,
                    session_id=session_id,
                    kwargs={}
                )

                response_queue = asyncio.Queue()
                STATE.pending_requests[request_id] = response_queue

                await app.ws_send(agent_conn_id, WsMessage(
                    event='run_request',
                    data=run_request.model_dump()
                ).model_dump())

                await app.ws_send(conn_id, {
                    "event": "message_acknowledged",
                    "data": {"request_id": request_id, "agent_id": agent_id}
                })

    except Exception as e:
        app.print(f"UI message handling error: {e}", error=True)
        await app.ws_send(conn_id, {
            "event": "error",
            "data": {"error": str(e)}
        })

types

AgentRegistered

Bases: BaseModel

Server -> Client: Response after successful registration.

Source code in toolboxv2/mods/registry/types.py

class AgentRegistered(BaseModel):
    """Server -> Client: Response after successful registration."""
    public_name: str
    public_agent_id: str = Field(..., description="The unique public ID for the agent.")
    public_api_key: str = Field(..., description="The secret API key for public access.")
    public_url: str = Field(..., description="The full public URL to run the agent.")

AgentRegistration

Bases: BaseModel

Client -> Server: Payload to register a new agent.

Source code in toolboxv2/mods/registry/types.py

class AgentRegistration(BaseModel):
    """Client -> Server: Payload to register a new agent."""
    public_name: str = Field(..., description="A user-friendly name for the agent.")
    description: str | None = Field(None, description="Optional description of the agent's capabilities.")

ExecutionError

Bases: BaseModel

Client -> Server: Reports an error during execution.

Source code in toolboxv2/mods/registry/types.py

class ExecutionError(BaseModel):
    """Client -> Server: Reports an error during execution."""
    request_id: str
    error: str

ExecutionResult

Bases: BaseModel

Client -> Server: A chunk of the execution result (for streaming).

Source code in toolboxv2/mods/registry/types.py

class ExecutionResult(BaseModel):
    """Client -> Server: A chunk of the execution result (for streaming)."""
    request_id: str
    payload: dict[str, Any] = Field(..., description="The ProgressEvent or final result as a dictionary.")
    is_final: bool = Field(False, description="True if this is the last message for this request.")

RunRequest

Bases: BaseModel

Server -> Client: Request to execute an agent.

Source code in toolboxv2/mods/registry/types.py

class RunRequest(BaseModel):
    """Server -> Client: Request to execute an agent."""
    request_id: str = Field(..., description="A unique ID for this specific execution request.")
    public_agent_id: str = Field(..., description="The ID of the agent to run.")
    query: str = Field(..., description="The main input/query for the agent.")
    session_id: str | None = Field(None, description="Session ID for maintaining context.")
    kwargs: dict[str, Any] = Field({}, description="Additional keyword arguments for the a_run method.")

WsMessage

Bases: BaseModel

A generic wrapper for all WebSocket messages.

Source code in toolboxv2/mods/registry/types.py

class WsMessage(BaseModel):
    """A generic wrapper for all WebSocket messages."""
    event: str
    data: dict[str, Any]

talk

TalkSession

Bases: BaseModel

Represents the state of a single voice conversation session.

Source code in toolboxv2/mods/talk.py

class TalkSession(BaseModel):
    """Represents the state of a single voice conversation session."""
    session_id: str = Field(default_factory=lambda: str(uuid.uuid4()))
    user_id: str
    chat_session: ChatSession
    event_queue: asyncio.Queue = Field(default_factory=asyncio.Queue, exclude=True)
    # Task to track the running agent process, preventing concurrent requests
    agent_task: asyncio.Task | None = Field(default=None, exclude=True)

    class Config:
        arbitrary_types_allowed = True

Tools

Bases: MainTool

The main class for the Talk module, handling initialization, session management, and dependency loading.

Source code in toolboxv2/mods/talk.py

class Tools(MainTool):
    """
    The main class for the Talk module, handling initialization,
    session management, and dependency loading.
    """

    def __init__(self, app: App):
        # Initialize the MainTool with module-specific information
        self.version = VERSION
        self.name = MOD_NAME
        self.color = "CYAN"
        self.sessions: dict[str, TalkSession] = {}
        self.stt_func = None
        self.tts_func = None
        self.isaa_mod = None
        super().__init__(load=self.on_start, v=VERSION, name=MOD_NAME, tool={}, on_exit=self.on_exit)

    def on_start(self):
        """Initializes the Talk module, its dependencies (ISAA, AUDIO), and UI registration."""
        self.app.logger.info(f"Starting {self.name} v{self.version}...")

        # Get the ISAA module instance, which is a critical dependency
        self.isaa_mod = None#self.app.get_mod("isaa")
        if not self.isaa_mod:
            self.app.logger.error(
                f"{self.name}: ISAA module not found or failed to load. Voice assistant will not be functional.")
            return

        # Initialize STT and TTS services from the AUDIO module
        if hasattr(TBEF, "AUDIO") and self.app.get_mod("AUDIO"):
            self.stt_func = self.app.run_any(TBEF.AUDIO.STT_GENERATE, model="openai/whisper-small", row=True, device=0)
            self.tts_func = self.app.get_function(TBEF.AUDIO.SPEECH, state=False)[0]

            if self.stt_func and self.stt_func != "404":
                self.app.logger.info("Talk STT (whisper-small) is Online.")
            else:
                self.app.logger.warning("Talk STT function not available.")
                self.stt_func = None

            if self.tts_func and self.tts_func != "404":
                self.app.logger.info("Talk TTS function is Online.")
            else:
                self.app.logger.warning("Talk TTS function not available.")
                self.tts_func = None
        else:
            self.app.logger.warning("Talk module: AUDIO module features are not available or the module is not loaded.")

        if not all([self.stt_func, self.tts_func]):
            self.app.logger.error("Talk module cannot function without both STT and TTS services.")

        # Register the UI component with CloudM
        self.app.run_any(("CloudM", "add_ui"),
                         name=MOD_NAME, title="Voice Assistant", path=f"/api/{MOD_NAME}/ui",
                         description="Natural conversation with an AI assistant.", auth=True)
        self.app.logger.info(f"{self.name} UI registered with CloudM.")

    def on_exit(self):
        """Clean up resources, especially cancelling any active agent tasks."""
        for session in self.sessions.values():
            if session.agent_task and not session.agent_task.done():
                session.agent_task.cancel()
        self.app.logger.info(f"Closing {self.name} and cleaning up sessions.")

on_exit()

Clean up resources, especially cancelling any active agent tasks.

Source code in toolboxv2/mods/talk.py

def on_exit(self):
    """Clean up resources, especially cancelling any active agent tasks."""
    for session in self.sessions.values():
        if session.agent_task and not session.agent_task.done():
            session.agent_task.cancel()
    self.app.logger.info(f"Closing {self.name} and cleaning up sessions.")

on_start()

Initializes the Talk module, its dependencies (ISAA, AUDIO), and UI registration.

Source code in toolboxv2/mods/talk.py

def on_start(self):
    """Initializes the Talk module, its dependencies (ISAA, AUDIO), and UI registration."""
    self.app.logger.info(f"Starting {self.name} v{self.version}...")

    # Get the ISAA module instance, which is a critical dependency
    self.isaa_mod = None#self.app.get_mod("isaa")
    if not self.isaa_mod:
        self.app.logger.error(
            f"{self.name}: ISAA module not found or failed to load. Voice assistant will not be functional.")
        return

    # Initialize STT and TTS services from the AUDIO module
    if hasattr(TBEF, "AUDIO") and self.app.get_mod("AUDIO"):
        self.stt_func = self.app.run_any(TBEF.AUDIO.STT_GENERATE, model="openai/whisper-small", row=True, device=0)
        self.tts_func = self.app.get_function(TBEF.AUDIO.SPEECH, state=False)[0]

        if self.stt_func and self.stt_func != "404":
            self.app.logger.info("Talk STT (whisper-small) is Online.")
        else:
            self.app.logger.warning("Talk STT function not available.")
            self.stt_func = None

        if self.tts_func and self.tts_func != "404":
            self.app.logger.info("Talk TTS function is Online.")
        else:
            self.app.logger.warning("Talk TTS function not available.")
            self.tts_func = None
    else:
        self.app.logger.warning("Talk module: AUDIO module features are not available or the module is not loaded.")

    if not all([self.stt_func, self.tts_func]):
        self.app.logger.error("Talk module cannot function without both STT and TTS services.")

    # Register the UI component with CloudM
    self.app.run_any(("CloudM", "add_ui"),
                     name=MOD_NAME, title="Voice Assistant", path=f"/api/{MOD_NAME}/ui",
                     description="Natural conversation with an AI assistant.", auth=True)
    self.app.logger.info(f"{self.name} UI registered with CloudM.")

api_open_stream(self, request, session_id) async

Opens a Server-Sent Events (SSE) stream for a given session ID.

Source code in toolboxv2/mods/talk.py

@export(mod_name=MOD_NAME, api=True, name="stream", api_methods=['GET'], request_as_kwarg=True)
async def api_open_stream(self: Tools, request: RequestData, session_id: str) -> Result:
    """Opens a Server-Sent Events (SSE) stream for a given session ID."""
    if not session_id or session_id not in self.sessions:
        return Result.default_user_error(info="Invalid or expired session ID.", exec_code=404)

    session = self.sessions[session_id]
    queue = session.event_queue

    async def event_generator() -> AsyncGenerator[dict[str, Any], None]:
        self.app.logger.info(f"SSE stream opened for session {session_id}")
        await queue.put({"event": "connection_ready", "data": "Stream connected successfully."})
        try:
            while True:
                event_data = await queue.get()
                yield event_data
                queue.task_done()
        except asyncio.CancelledError:
            self.app.logger.info(f"SSE stream for session {session_id} cancelled by client.")
        finally:
            if session_id in self.sessions:
                if self.sessions[session_id].agent_task and not self.sessions[session_id].agent_task.done():
                    self.sessions[session_id].agent_task.cancel()
                del self.sessions[session_id]
                self.app.logger.info(f"Cleaned up and closed session {session_id}.")

    return Result.sse(stream_generator=event_generator())

api_process_audio(self, request, form_data) async

Receives audio, transcribes it, and starts the agent processing task.

Source code in toolboxv2/mods/talk.py

@export(mod_name=MOD_NAME, api=True, name="process_audio", api_methods=['POST'], request_as_kwarg=True)
async def api_process_audio(self: Tools, request: RequestData, form_data: dict) -> Result:
    """Receives audio, transcribes it, and starts the agent processing task."""
    if not self.stt_func:
        return Result.default_internal_error(info="Speech-to-text service is not available.")

    session_id = form_data.get('session_id')
    audio_file_data = form_data.get('audio_blob')

    if not session_id or session_id not in self.sessions:
        return Result.default_user_error(info="Invalid or missing session_id.", exec_code=400)

    session = self.sessions[session_id]

    if session.agent_task and not session.agent_task.done():
        return Result.default_user_error(info="Already processing a previous request.", exec_code=429)

    if not audio_file_data or 'content_base64' not in audio_file_data:
        return Result.default_user_error(info="Audio data is missing or in the wrong format.", exec_code=400)

    try:
        audio_bytes = base64.b64decode(audio_file_data['content_base64'])
        transcription_result = self.stt_func(audio_bytes)
        transcribed_text = transcription_result.get('text', '').strip()

        if not transcribed_text:
            await session.event_queue.put({"event": "error", "data": "Could not understand audio. Please try again."})
            return Result.ok(data={"message": "Transcription was empty."})

        await session.event_queue.put({"event": "transcription_update", "data": transcribed_text})

        voice_params = {
            "voice_index": int(form_data.get('voice_index', '0')),
            "provider": form_data.get('provider', 'piper'),
            "model_name": form_data.get('model_name', 'ryan')
        }

        # Start the background task; the request returns immediately.
        session.agent_task = asyncio.create_task(
            _run_agent_and_respond(self, session, transcribed_text, voice_params)
        )
        return Result.ok(data={"message": "Audio received and processing started."})

    except Exception as e:
        self.app.logger.error(f"Error processing audio for session {session_id}: {e}", exc_info=True)
        return Result.default_internal_error(info=f"Failed to process audio: {str(e)}")

api_start_session(self, request) async

Creates a new talk session for an authenticated user.

Source code in toolboxv2/mods/talk.py

@export(mod_name=MOD_NAME, api=True, name="start_session", api_methods=['POST'], request_as_kwarg=True)
async def api_start_session(self: Tools, request: RequestData) -> Result:
    """Creates a new talk session for an authenticated user."""
    user_id = await _get_user_uid(self.app, request)
    if not user_id:
        return Result.default_user_error(info="User authentication required.", exec_code=401)

    if not self.isaa_mod:
        return Result.default_internal_error(info="ISAA module is not available.")

    # Create a new ISAA ChatSession for conversation history
    chat_session = ChatSession(mem=self.isaa_mod.get_memory())
    session = TalkSession(user_id=user_id, chat_session=chat_session)
    self.sessions[session.session_id] = session

    self.app.logger.info(f"Started new talk session {session.session_id} for user {user_id}")
    return Result.json(data={"session_id": session.session_id})

get_main_ui(self, request)

Serves the main HTML and JavaScript UI for the Talk widget.

Source code in toolboxv2/mods/talk.py

@export(mod_name=MOD_NAME, name="ui", api=True, api_methods=['GET'], request_as_kwarg=True)
def get_main_ui(self: Tools, request: RequestData) -> Result:
    """Serves the main HTML and JavaScript UI for the Talk widget."""
    html_content = """
<!DOCTYPE html>
<html lang="en" data-theme="light">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>ToolBoxV2 - Voice Assistant</title>
    <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Material+Symbols+Outlined:opsz,wght,FILL,GRAD@20..48,100..700,0..1,-50..200" />
    <style>
        body { font-family: sans-serif; background-color: var(--theme-bg); color: var(--theme-text); display: flex; justify-content: center; align-items: center; min-height: 100vh; margin: 0; }
        .container { display: flex; flex-direction: column; align-items: center; justify-content: center; width: 100%; max-width: 600px; padding: 20px; text-align: center; }
        .visualizer { width: 250px; height: 250px; background-color: var(--glass-bg); border-radius: 50%; position: relative; overflow: hidden; border: 3px solid var(--theme-border); box-shadow: inset 0 0 15px rgba(0,0,0,0.2); transition: border-color 0.3s, box-shadow 0.3s; }
        .visualizer.recording { border-color: #ef4444; }
        .visualizer.thinking { border-color: #3b82f6; animation: pulse 2s infinite; }
        .visualizer.speaking { border-color: #22c55e; }
        .particle { position: absolute; width: 8px; height: 8px; background-color: var(--theme-primary); border-radius: 50%; pointer-events: none; transition: all 0.1s; }
        #micButton { margin-top: 30px; width: 80px; height: 80px; border-radius: 50%; border: none; background-color: var(--theme-primary); color: white; cursor: pointer; display: flex; justify-content: center; align-items: center; box-shadow: 0 4px 10px rgba(0,0,0,0.2); transition: background-color 0.2s, transform 0.1s; }
        #micButton:active { transform: scale(0.95); }
        #micButton:disabled { background-color: #9ca3af; cursor: not-allowed; }
        #micButton .material-symbols-outlined { font-size: 40px; }
        #statusText { margin-top: 20px; min-height: 50px; font-size: 1.2em; color: var(--theme-text-muted); line-height: 1.5; }
        @keyframes pulse { 0% { box-shadow: inset 0 0 15px rgba(0,0,0,0.2), 0 0 0 0 rgba(59, 130, 246, 0.7); } 70% { box-shadow: inset 0 0 15px rgba(0,0,0,0.2), 0 0 0 15px rgba(59, 130, 246, 0); } 100% { box-shadow: inset 0 0 15px rgba(0,0,0,0.2), 0 0 0 0 rgba(59, 130, 246, 0); } }
    </style>
</head>
<body>
    <div class="container">
        <div class="visualizer" id="visualizer"></div>
        <p id="statusText">Press the microphone to start</p>
        <button id="micButton"><span class="material-symbols-outlined">hourglass_empty</span></button>
        <div class="options" style="margin-top: 20px;">
            <label for="voiceSelect">Voice:</label>
            <select id="voiceSelect">
                <option value='{"provider": "piper", "model_name": "ryan", "voice_index": 0}'>Ryan (EN)</option>
                <option value='{"provider": "piper", "model_name": "kathleen", "voice_index": 0}'>Kathleen (EN)</option>
                <option value='{"provider": "piper", "model_name": "karlsson", "voice_index": 0}'>Karlsson (DE)</option>
            </select>
        </div>
    </div>
    <script unSave="true">
    function initTalk() {
        const visualizer = document.getElementById('visualizer');
        const micButton = document.getElementById('micButton');
        const statusText = document.getElementById('statusText');
        const voiceSelect = document.getElementById('voiceSelect');

        const state = { sessionId: null, sseConnection: null, mediaRecorder: null, audioChunks: [], isRecording: false, isProcessing: false, currentAudio: null };
        let audioContext, analyser, particles = [];

        function setStatus(text, mode = 'idle') {
            statusText.textContent = text;
            visualizer.className = 'visualizer ' + mode;
        }

        function createParticles(num = 50) {
            visualizer.innerHTML = ''; particles = [];
            for (let i = 0; i < num; i++) {
                const p = document.createElement('div'); p.classList.add('particle');
                visualizer.appendChild(p);
                particles.push({ element: p, angle: Math.random() * Math.PI * 2, radius: 50 + Math.random() * 50, speed: 0.01 + Math.random() * 0.02 });
            }
        }

        function animateVisualizer() {
            if (analyser) {
                const dataArray = new Uint8Array(analyser.frequencyBinCount);
                analyser.getByteFrequencyData(dataArray);
                let average = dataArray.reduce((a, b) => a + b, 0) / dataArray.length;
                particles.forEach(p => {
                    p.angle += p.speed;
                    const scale = 1 + (average / 128);
                    p.element.style.transform = `translate(${Math.cos(p.angle) * p.radius * scale}px, ${Math.sin(p.angle) * p.radius * scale}px)`;
                });
            }
            requestAnimationFrame(animateVisualizer);
        }

        async function startSession() {
            if (state.sessionId) return;
            setStatus("Connecting...", 'thinking');
            micButton.disabled = true;
            try {
                const response = await TB.api.request('talk', 'start_session', {}, 'POST');
                if (response.error === 'none' && response.get()?.session_id) {
                    state.sessionId = response.get().session_id;
                    connectSse();
                } else {
                    setStatus(response.info?.help_text || "Failed to start session.", 'error');
                }
            } catch (e) {
                setStatus("Connection error.", 'error');
            }
        }

        function connectSse() {
            if (!state.sessionId) return;
            state.sseConnection = TB.sse.connect(`/sse/talk/stream?session_id=${state.sessionId}`, {
                onOpen: () => console.log("SSE Stream Open"),
                onError: () => setStatus("Connection lost.", 'error'),
                listeners: {
                    'connection_ready': (data) => { setStatus("Press the microphone to start"); micButton.disabled = false; micButton.innerHTML = '<span class="material-symbols-outlined">mic</span>'; },
                    'transcription_update': (data) => { setStatus(`“${data}”`, 'thinking'); state.isProcessing = true; },
                    'agent_thought': (data) => setStatus(data, 'thinking'),
                    'agent_response_chunk': (data) => { if (statusText.textContent.startsWith('“')) statusText.textContent = ""; statusText.textContent += data; },
                    'audio_playback': (data) => playAudio(data.content, data.format),
                    'processing_complete': (data) => { state.isProcessing = false; setStatus(data); micButton.disabled = false; micButton.innerHTML = '<span class="material-symbols-outlined">mic</span>'; },
                    'error': (data) => { state.isProcessing = false; setStatus(data, 'error'); micButton.disabled = false; micButton.innerHTML = '<span class="material-symbols-outlined">mic</span>'; }
                }
            });
        }

        async function playAudio(base64, format) {
            setStatus("...", 'speaking');
            const blob = await (await fetch(`data:${format};base64,${base64}`)).blob();
            const url = URL.createObjectURL(blob);
            if (state.currentAudio) state.currentAudio.pause();
            state.currentAudio = new Audio(url);

            if (!audioContext) audioContext = new AudioContext();
            const source = audioContext.createMediaElementSource(state.currentAudio);
            if (!analyser) { analyser = audioContext.createAnalyser(); analyser.fftSize = 64; }
            source.connect(analyser);
            analyser.connect(audioContext.destination);

            state.currentAudio.play();
            state.currentAudio.onended = () => { setStatus("Finished speaking."); URL.revokeObjectURL(url); };
        }

        async function toggleRecording() {
            if (state.isProcessing) return;
            if (!state.sessionId) { await startSession(); return; }

            if (state.isRecording) {
                state.mediaRecorder.stop();
                micButton.disabled = true;
                micButton.innerHTML = '<span class="material-symbols-outlined">hourglass_top</span>';
                setStatus("Processing...", 'thinking');
            } else {
                if (!state.mediaRecorder) {
                    try {
                        const stream = await navigator.mediaDevices.getUserMedia({ audio: { sampleRate: 16000, channelCount: 1 } });
                        if (!audioContext) audioContext = new AudioContext();
                        const source = audioContext.createMediaStreamSource(stream);
                        if (!analyser) { analyser = audioContext.createAnalyser(); analyser.fftSize = 64; }
                        source.connect(analyser);

                        state.mediaRecorder = new MediaRecorder(stream, { mimeType: 'audio/webm;codecs=opus' });
                        state.mediaRecorder.ondataavailable = e => state.audioChunks.push(e.data);
                        state.mediaRecorder.onstop = uploadAudio;
                    } catch (e) { setStatus("Could not access microphone.", 'error'); return; }
                }
                state.audioChunks = []; state.mediaRecorder.start(); state.isRecording = true;
                setStatus("Listening...", 'recording');
                micButton.innerHTML = '<span class="material-symbols-outlined">stop_circle</span>';
            }
        }

        async function uploadAudio() {
            state.isRecording = false; state.isProcessing = true;
            if (state.audioChunks.length === 0) { setStatus("No audio recorded."); state.isProcessing = false; micButton.disabled = false; micButton.innerHTML = '<span class="material-symbols-outlined">mic</span>'; return; }
            const audioBlob = new Blob(state.audioChunks, { type: 'audio/webm;codecs=opus' });

            const formData = new FormData();
            formData.append('session_id', state.sessionId);
            formData.append('audio_blob', audioBlob, 'recording.webm');

            const voiceParams = JSON.parse(voiceSelect.value);
            for (const key in voiceParams) formData.append(key, voiceParams[key]);

            try {
                const response = await TB.api.request('talk', 'process_audio', formData, 'POST');
                if (response.error !== 'none') {
                    setStatus(response.info?.help_text || "Failed to process audio.", 'error');
                    state.isProcessing = false; micButton.disabled = false; micButton.innerHTML = '<span class="material-symbols-outlined">mic</span>';
                }
            } catch(e) {
                 setStatus("Error sending audio.", 'error'); state.isProcessing = false; micButton.disabled = false; micButton.innerHTML = '<span class="material-symbols-outlined">mic</span>';
            }
        }

        micButton.addEventListener('click', toggleRecording);
        createParticles(); animateVisualizer();
        if (window.TB.isInitialized) startSession(); else window.TB.events.on('tbjs:initialized', startSession, { once: true });
    }
if (window.TB?.events) {
    if (window.TB.config?.get('appRootId')) { // A sign that TB.init might have run
         initTalk();
    } else {
        window.TB.events.on('tbjs:initialized', initTalk, { once: true });
    }
} else {
    // Fallback if TB is not even an object yet, very early load
    document.addEventListener('tbjs:initialized', initTalk, { once: true }); // Custom event dispatch from TB.init
}

    </script>
</body>
</html>"""
    return Result.html(data=html_content)

toolboxv2.flows_dict(s='.py', remote=False, dir_path=None, flows_dict_=None)

Source code in toolboxv2/flows/__init__.py

def flows_dict(s='.py', remote=False, dir_path=None, flows_dict_=None):

    if flows_dict_ is None:
        flows_dict_ = {}
    with Spinner("Loading flows"):
        # Erhalte den Pfad zum aktuellen Verzeichnis
        if dir_path is None:
            for ex_path in os.getenv("EXTERNAL_PATH_RUNNABLE", '').split(','):
                if not ex_path or len(ex_path) == 0:
                    continue
                flows_dict(s,remote,ex_path,flows_dict_)
            dir_path = os.path.dirname(os.path.realpath(__file__))
        to = time.perf_counter()
        # Iteriere über alle Dateien im Verzeichnis
        files = os.listdir(dir_path)
        l_files = len(files)
        for i, file_name in enumerate(files):
            if not file_name:
                continue
            with Spinner(f"{file_name} {i}/{l_files}"):
                if file_name == "__init__.py":
                    pass

                elif remote and s in file_name and file_name.endswith('.gist'):
                    name_f = os.path.splitext(file_name)[0]
                    name = name_f.split('.')[0]
                    url = name_f.split('.')[-1]
                    try:
                        module = GistLoader(f"{name}/{url}").load_module(name)
                    except Exception as e:
                        continue

                    if hasattr(module, 'run') and callable(module.run) and hasattr(module, 'NAME'):
                        flows_dict_[module.NAME] = module.run
                elif file_name.endswith('.py') and s in file_name:
                    name = os.path.splitext(file_name)[0]
                    spec = importlib.util.spec_from_file_location(name, os.path.join(dir_path, file_name))
                    module = importlib.util.module_from_spec(spec)
                    try:
                        spec.loader.exec_module(module)
                    except Exception:
                        continue

                    # Füge das Modul der Dictionary hinzu
                    if hasattr(module, 'run') and callable(module.run) and hasattr(module, 'NAME'):
                        flows_dict_[module.NAME] = module.run

        return flows_dict_

toolboxv2.TBEF

Automatic generated by ToolBox v = 0.1.22

Other Exposed Items

toolboxv2.ToolBox_over = 'root' module-attribute